public static TableDesc getTableDesc(String cols, String colTypes) { return (new TableDesc(SequenceFileInputFormat.class, HiveSequenceFileOutputFormat.class, Utilities.makeProperties( serdeConstants.SERIALIZATION_FORMAT, "" + Utilities.ctrlaCode, serdeConstants.LIST_COLUMNS, cols, serdeConstants.LIST_COLUMN_TYPES, colTypes, serdeConstants.SERIALIZATION_LIB,LazySimpleSerDe.class.getName()))); }
public static TableDesc getTableDesc(String cols, String colTypes) { return (new TableDesc(SequenceFileInputFormat.class, HiveSequenceFileOutputFormat.class, Utilities.makeProperties( serdeConstants.SERIALIZATION_FORMAT, "" + Utilities.ctrlaCode, serdeConstants.LIST_COLUMNS, cols, serdeConstants.LIST_COLUMN_TYPES, colTypes, serdeConstants.SERIALIZATION_LIB,LazySimpleSerDe.class.getName()))); }
/** * Generate the table descriptor of MetadataTypedColumnsetSerDe with the * separatorCode. MetaDataTypedColumnsetSerDe is used because LazySimpleSerDe * does not support a table with a single column "col" with type * "array<string>". */ public static TableDesc getDefaultTableDesc(String separatorCode) { return new TableDesc( TextInputFormat.class, IgnoreKeyTextOutputFormat.class, Utilities .makeProperties( org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT,separatorCode, serdeConstants.SERIALIZATION_LIB,MetadataTypedColumnsetSerDe.class.getName())); }
public static TableDesc createTableDesc(StructObjectInspector oI) { Map<String,String> props = new HashMap<String,String>(); PTFDeserializer.addOIPropertiestoSerDePropsMap(oI, props); String colNames = props.get(serdeConstants.LIST_COLUMNS); String colTypes = props.get(serdeConstants.LIST_COLUMN_TYPES); TableDesc tblDesc = new TableDesc( PTFSequenceFileInputFormat.class, PTFHiveSequenceFileOutputFormat.class, Utilities.makeProperties( serdeConstants.SERIALIZATION_FORMAT, ""+ Utilities.ctrlaCode, serdeConstants.LIST_COLUMNS, colNames.toString(), serdeConstants.LIST_COLUMN_TYPES,colTypes.toString(), serdeConstants.SERIALIZATION_LIB,LazyBinarySerDe.class.getName())); return tblDesc; }
public static TableDesc createTableDesc(StructObjectInspector oI) { Map<String,String> props = new HashMap<String,String>(); PTFDeserializer.addOIPropertiestoSerDePropsMap(oI, props); String colNames = props.get(serdeConstants.LIST_COLUMNS); String colTypes = props.get(serdeConstants.LIST_COLUMN_TYPES); TableDesc tblDesc = new TableDesc( PTFSequenceFileInputFormat.class, PTFHiveSequenceFileOutputFormat.class, Utilities.makeProperties( serdeConstants.SERIALIZATION_FORMAT, ""+ Utilities.ctrlaCode, serdeConstants.LIST_COLUMNS, colNames.toString(), serdeConstants.LIST_COLUMN_TYPES,colTypes.toString(), serdeConstants.SERIALIZATION_LIB,LazyBinarySerDe.class.getName())); return tblDesc; }
@Override public Object clone() { TableDesc ret = new TableDesc(); ret.setInputFileFormatClass(inputFileFormatClass); ret.setOutputFileFormatClass(outputFileFormatClass); Properties newProp = new Properties(); Enumeration<Object> keysProp = properties.keys(); while (keysProp.hasMoreElements()) { Object key = keysProp.nextElement(); newProp.put(key, properties.get(key)); } ret.setProperties(newProp); if (jobProperties != null) { ret.jobProperties = new LinkedHashMap<String, String>(jobProperties); } return ret; }
/** * Create a FetchTask for a given schema. * * @param schema string */ protected FetchTask createFetchTask(String schema) { Properties prop = new Properties(); // Sets delimiter to tab (ascii 9) prop.setProperty(serdeConstants.SERIALIZATION_FORMAT, Integer.toString(Utilities.tabCode)); prop.setProperty(serdeConstants.SERIALIZATION_NULL_FORMAT, " "); String[] colTypes = schema.split("#"); prop.setProperty("columns", colTypes[0]); prop.setProperty("columns.types", colTypes[1]); prop.setProperty(serdeConstants.SERIALIZATION_LIB, LazySimpleSerDe.class.getName()); FetchWork fetch = new FetchWork(ctx.getResFile(), new TableDesc(TextInputFormat.class, IgnoreKeyTextOutputFormat.class, prop), -1); fetch.setSerializationNullFormat(" "); return (FetchTask) TaskFactory.get(fetch); }
public static TableDesc getTableDesc(Table tbl) { Properties props = tbl.getMetadata(); props.put(serdeConstants.SERIALIZATION_LIB, tbl.getDeserializer().getClass().getName()); return (new TableDesc(tbl.getInputFormatClass(), tbl .getOutputFormatClass(), props)); }
@Override public Object clone() { TableDesc ret = new TableDesc(); ret.setInputFileFormatClass(inputFileFormatClass); ret.setOutputFileFormatClass(outputFileFormatClass); Properties newProp = new Properties(); Enumeration<Object> keysProp = properties.keys(); while (keysProp.hasMoreElements()) { Object key = keysProp.nextElement(); newProp.put(key, properties.get(key)); } ret.setProperties(newProp); if (jobProperties != null) { ret.jobProperties = new LinkedHashMap<String, String>(jobProperties); } return ret; }
/** * Generate the table descriptor for intermediate files. */ public static TableDesc getReduceValueTableDesc(List<FieldSchema> fieldSchemas) { return new TableDesc(SequenceFileInputFormat.class, SequenceFileOutputFormat.class, Utilities.makeProperties( serdeConstants.LIST_COLUMNS, MetaStoreUtils .getColumnNamesFromFieldSchema(fieldSchemas), serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), serdeConstants.ESCAPE_CHAR, "\\", serdeConstants.SERIALIZATION_LIB,LazyBinarySerDe.class.getName())); }
public static TableDesc getTableDesc(Table tbl) { Properties props = tbl.getMetadata(); props.put(serdeConstants.SERIALIZATION_LIB, tbl.getDeserializer().getClass().getName()); return (new TableDesc(tbl.getInputFormatClass(), tbl .getOutputFormatClass(), props)); }
/** * Generate the table descriptor for intermediate files. */ public static TableDesc getReduceValueTableDesc(List<FieldSchema> fieldSchemas) { return new TableDesc(SequenceFileInputFormat.class, SequenceFileOutputFormat.class, Utilities.makeProperties( serdeConstants.LIST_COLUMNS, MetaStoreUtils .getColumnNamesFromFieldSchema(fieldSchemas), serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), serdeConstants.ESCAPE_CHAR, "\\", serdeConstants.SERIALIZATION_LIB,LazyBinarySerDe.class.getName())); }
@BeforeClass public static void classSetup() { Properties properties = new Properties(); properties.setProperty(serdeConstants.SERIALIZATION_LIB, TFSOSerDe.class.getName()); properties.setProperty(hive_metastoreConstants.META_TABLE_NAME, "tfs"); nonAcidTableDescriptor = new TableDesc(TFSOInputFormat.class, TFSOOutputFormat.class, properties); properties.setProperty(serdeConstants.LIST_COLUMNS,"data"); properties = new Properties(properties); properties.setProperty(hive_metastoreConstants.BUCKET_COUNT, "1"); acidTableDescriptor = new TableDesc(TFSOInputFormat.class, TFSOOutputFormat.class, properties); tmpdir = new File(System.getProperty("java.io.tmpdir") + System.getProperty("file.separator") + "testFileSinkOperator"); tmpdir.mkdir(); tmpdir.deleteOnExit(); }
/** * Generate the table descriptor for Map-side join value. */ public static TableDesc getMapJoinValueTableDesc( List<FieldSchema> fieldSchemas) { return new TableDesc(SequenceFileInputFormat.class, SequenceFileOutputFormat.class, Utilities.makeProperties( serdeConstants.LIST_COLUMNS, MetaStoreUtils .getColumnNamesFromFieldSchema(fieldSchemas), serdeConstants.COLUMN_NAME_DELIMITER, MetaStoreUtils.getColumnNameDelimiter(fieldSchemas), serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), serdeConstants.ESCAPE_CHAR, "\\", serdeConstants.SERIALIZATION_LIB,LazyBinarySerDe.class.getName())); }
/** * Generate the table descriptor for reduce key. */ public static TableDesc getReduceKeyTableDesc(List<FieldSchema> fieldSchemas, String order, String nullOrder) { return new TableDesc( SequenceFileInputFormat.class, SequenceFileOutputFormat.class, Utilities.makeProperties(serdeConstants.LIST_COLUMNS, MetaStoreUtils .getColumnNamesFromFieldSchema(fieldSchemas), serdeConstants.COLUMN_NAME_DELIMITER, MetaStoreUtils.getColumnNameDelimiter(fieldSchemas), serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), serdeConstants.SERIALIZATION_SORT_ORDER, order, serdeConstants.SERIALIZATION_NULL_SORT_ORDER, nullOrder, serdeConstants.SERIALIZATION_LIB, BinarySortableSerDe.class.getName())); }
/** * Generate the table descriptor for Map-side join value. */ public static TableDesc getMapJoinValueTableDesc( List<FieldSchema> fieldSchemas) { return new TableDesc(SequenceFileInputFormat.class, SequenceFileOutputFormat.class, Utilities.makeProperties( serdeConstants.LIST_COLUMNS, MetaStoreUtils .getColumnNamesFromFieldSchema(fieldSchemas), serdeConstants.COLUMN_NAME_DELIMITER, MetaStoreUtils.getColumnNameDelimiter(fieldSchemas), serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), serdeConstants.ESCAPE_CHAR, "\\", serdeConstants.SERIALIZATION_LIB,LazyBinarySerDe.class.getName())); }
/** * Generate the table descriptor for reduce key. */ public static TableDesc getReduceKeyTableDesc(List<FieldSchema> fieldSchemas, String order, String nullOrder) { return new TableDesc( SequenceFileInputFormat.class, SequenceFileOutputFormat.class, Utilities.makeProperties(serdeConstants.LIST_COLUMNS, MetaStoreUtils .getColumnNamesFromFieldSchema(fieldSchemas), serdeConstants.COLUMN_NAME_DELIMITER, MetaStoreUtils.getColumnNameDelimiter(fieldSchemas), serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), serdeConstants.SERIALIZATION_SORT_ORDER, order, serdeConstants.SERIALIZATION_NULL_SORT_ORDER, nullOrder, serdeConstants.SERIALIZATION_LIB, BinarySortableSerDe.class.getName())); }
/** * Generate the table descriptor for intermediate files. */ public static TableDesc getIntermediateFileTableDesc( List<FieldSchema> fieldSchemas) { return new TableDesc(SequenceFileInputFormat.class, SequenceFileOutputFormat.class, Utilities.makeProperties( serdeConstants.LIST_COLUMNS, MetaStoreUtils .getColumnNamesFromFieldSchema(fieldSchemas), serdeConstants.COLUMN_NAME_DELIMITER, MetaStoreUtils.getColumnNameDelimiter(fieldSchemas), serdeConstants.COLUMN_NAME_DELIMITER, MetaStoreUtils.getColumnNameDelimiter(fieldSchemas), serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), serdeConstants.ESCAPE_CHAR, "\\", serdeConstants.SERIALIZATION_LIB,LazyBinarySerDe.class.getName())); }
private FileSinkOperator createFileSinkOperator(Path finalDirName) { FileSinkOperator fileSinkOperator = mock(FileSinkOperator.class); TableDesc tableDesc = new TableDesc(HiveInputFormat.class, HiveOutputFormat.class, new Properties()); FileSinkDesc fileSinkDesc = new FileSinkDesc(finalDirName, tableDesc, false); fileSinkDesc.setDirName(finalDirName); when(fileSinkOperator.getConf()).thenReturn(fileSinkDesc); when(fileSinkOperator.getSchema()).thenReturn(mock(RowSchema.class)); fileSinkDesc.setTableInfo(tableDesc); when(fileSinkOperator.getCompilationOpContext()).thenReturn(mock(CompilationOpContext.class)); return fileSinkOperator; }
/** * Generate the table descriptor for intermediate files. */ public static TableDesc getIntermediateFileTableDesc( List<FieldSchema> fieldSchemas) { return new TableDesc(SequenceFileInputFormat.class, SequenceFileOutputFormat.class, Utilities.makeProperties( serdeConstants.LIST_COLUMNS, MetaStoreUtils .getColumnNamesFromFieldSchema(fieldSchemas), serdeConstants.COLUMN_NAME_DELIMITER, MetaStoreUtils.getColumnNameDelimiter(fieldSchemas), serdeConstants.COLUMN_NAME_DELIMITER, MetaStoreUtils.getColumnNameDelimiter(fieldSchemas), serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils .getColumnTypesFromFieldSchema(fieldSchemas), serdeConstants.ESCAPE_CHAR, "\\", serdeConstants.SERIALIZATION_LIB,LazyBinarySerDe.class.getName())); }