@Override public String getTableName() { return flatDesc.getTableName(); }
private static void appendWhereStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, boolean singleLine) { final String sep = singleLine ? " " : "\n"; StringBuilder whereBuilder = new StringBuilder(); whereBuilder.append("WHERE 1=1"); DataModelDesc model = flatDesc.getDataModel(); if (StringUtils.isNotEmpty(model.getFilterCondition())) { String quotedFilterCondition = quoteIdentifierInSqlExpr(flatDesc, model.getFilterCondition(), "`"); whereBuilder.append(" AND (").append(quotedFilterCondition).append(") "); // -> filter condition contains special character may cause bug } if (flatDesc.getSegment() != null) { PartitionDesc partDesc = model.getPartitionDesc(); if (partDesc != null && partDesc.getPartitionDateColumn() != null) { SegmentRange segRange = flatDesc.getSegRange(); if (segRange != null && !segRange.isInfinite()) { whereBuilder.append(" AND ("); String quotedPartitionCond = quoteIdentifierInSqlExpr(flatDesc, partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc, flatDesc.getSegment(), segRange), "`"); whereBuilder.append(quotedPartitionCond); whereBuilder.append(")" + sep); } } } sql.append(whereBuilder.toString()); }
public static String generateCreateTableStatement(IJoinedFlatTableDesc flatDesc, String storageDfsDir, String storageFormat, String fieldDelimiter) { StringBuilder ddl = new StringBuilder(); ddl.append("CREATE EXTERNAL TABLE IF NOT EXISTS " + flatDesc.getTableName() + "\n"); ddl.append("(" + "\n"); for (int i = 0; i < flatDesc.getAllColumns().size(); i++) { TblColRef col = flatDesc.getAllColumns().get(i); if (i > 0) { ddl.append(","); } ddl.append(colName(col, flatDesc.useAlias()) + " " + getHiveDataType(col.getDatatype()) + "\n"); } ddl.append(")" + "\n"); if (TEXTFILE.equals(storageFormat)) { ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY '" + fieldDelimiter + "'\n"); } ddl.append("STORED AS " + storageFormat + "\n"); ddl.append("LOCATION '" + getTableDir(flatDesc, storageDfsDir) + "';").append("\n"); ddl.append("ALTER TABLE " + flatDesc.getTableName() + " SET TBLPROPERTIES('auto.purge'='true');\n"); return ddl.toString(); }
public static String generateRedistributeFlatTableStatement(IJoinedFlatTableDesc flatDesc, CubeDesc cubeDesc) { final String tableName = flatDesc.getTableName(); StringBuilder sql = new StringBuilder(); sql.append("INSERT OVERWRITE TABLE " + quoteIdentifier(tableName) + " SELECT * FROM " + quoteIdentifier(tableName)); if (flatDesc.getClusterBy() != null) { appendClusterStatement(sql, flatDesc.getClusterBy()); } else if (flatDesc.getDistributedBy() != null) { appendDistributeStatement(sql, Lists.newArrayList(flatDesc.getDistributedBy())); } else { int redistColumnCount = cubeDesc.getConfig().getHiveRedistributeColumnCount(); RowKeyColDesc[] rowKeyColDescs = cubeDesc.getRowkey().getRowKeyColumns(); if (rowKeyColDescs.length < redistColumnCount) redistColumnCount = rowKeyColDescs.length; List<TblColRef> redistColumns = Lists.newArrayListWithCapacity(redistColumnCount); for (int i = 0; i < redistColumnCount; i++) { redistColumns.add(rowKeyColDescs[i].getColRef()); } appendDistributeStatement(sql, redistColumns); } return sql.toString(); }
public static String generateInsertDataStatement(IJoinedFlatTableDesc flatDesc) { CubeSegment segment = ((CubeSegment) flatDesc.getSegment()); KylinConfig kylinConfig; if (null == segment) { kylinConfig = KylinConfig.getInstanceFromEnv(); } else { kylinConfig = (flatDesc.getSegment()).getConfig(); } if (kylinConfig.isAdvancedFlatTableUsed()) { try { Class advancedFlatTable = Class.forName(kylinConfig.getAdvancedFlatTableClass()); Method method = advancedFlatTable.getMethod("generateInsertDataStatement", IJoinedFlatTableDesc.class, JobEngineConfig.class); return (String) method.invoke(null, flatDesc); } catch (Exception e) { throw new RuntimeException(e); } } return "INSERT OVERWRITE TABLE " + quoteIdentifier(flatDesc.getTableName()) + " " + generateSelectDataStatement(flatDesc) + ";\n"; }
protected AbstractExecutable createSqoopToFlatHiveStep(String jobWorkingDir, String cubeName) { KylinConfig config = getConfig(); PartitionDesc partitionDesc = flatDesc.getDataModel().getPartitionDesc(); String partCol = null; String hiveTable = flatDesc.getTableName(); String connectionUrl = config.getJdbcSourceConnectionUrl(); String driverClass = config.getJdbcSourceDriver(); splitColumn, splitDatabase, splitTable, splitTableAlias); if (partitionDesc.isPartitioned()) { SegmentRange segRange = flatDesc.getSegRange(); if (segRange != null && !segRange.isInfinite()) { if (partitionDesc.getPartitionDateColumnRef().getTableAlias().equals(splitTableAlias) String quotedPartCond = FlatTableSqlQuoteUtils.quoteIdentifierInSqlExpr(flatDesc, partitionDesc.getPartitionConditionBuilder().buildDateRangeCondition(partitionDesc, flatDesc.getSegment(), segRange), "`"); bquery += " WHERE " + quotedPartCond;
if (null != flatDesc.getClusterBy()) { return flatDesc.getClusterBy(); if (null != flatDesc.getDistributedBy()) { return flatDesc.getDistributedBy(); PartitionDesc partitionDesc = flatDesc.getDataModel().getPartitionDesc(); if (partitionDesc.isPartitioned()) { return partitionDesc.getPartitionDateColumnRef(); TableMetadataManager tblManager = TableMetadataManager.getInstance(getConfig()); long maxCardinality = 0; for (TableRef tableRef : flatDesc.getDataModel().getAllTables()) { TableExtDesc tableExtDesc = tblManager.getTableExt(tableRef.getTableDesc()); List<TableExtDesc.ColumnStats> columnStatses = tableExtDesc.getColumnStats(); for (TblColRef colRef : flatDesc.getAllColumns()) { if (colRef.getType().isIntegerFamily()) { return colRef; splitColumn = flatDesc.getAllColumns().get(0);
@Override public DataModelDesc getDataModel() { return flatDesc.getDataModel(); }
final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatDesc, baseLocation); String insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatDesc); insertDataHqls = insertDataHqls.replace(flatDesc.getDataModel().getRootFactTableName() + " ", mockFactTableName + " "); step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE); intermediateTables.add(flatDesc.getTableName()); intermediateTables.add(mockFactTableName); intermediatePaths.add(baseLocation + "/" + flatDesc.getTableName()); intermediatePaths.add(baseLocation + "/" + mockFactTableName); return step;
step.setName(ExecutableConstants.STEP_NAME_MATERIALIZE_HIVE_VIEW_IN_LOOKUP); KylinConfig kylinConfig = flatDesc.getSegment().getConfig(); TableMetadataManager metadataManager = TableMetadataManager.getInstance(kylinConfig); final Set<TableDesc> lookupViewsTables = Sets.newHashSet(); String prj = flatDesc.getDataModel().getProject(); for (JoinTableDesc lookupDesc : flatDesc.getDataModel().getJoinTables()) { TableDesc tableDesc = metadataManager.getTableDesc(lookupDesc.getTable(), prj); if (lookupDesc.getKind() == DataModelDesc.TableKind.LOOKUP && tableDesc.isView()) {
@Override public List<TblColRef> getAllColumns() { return flatDesc.getAllColumns(); }
@Override public ISegment getSegment() { return flatDesc.getSegment(); }
@Override public TblColRef getClusterBy() { return flatDesc.getClusterBy(); }
@Override public SegmentRange getSegRange() { return flatDesc.getSegRange(); }
@Override public TblColRef getDistributedBy() { return flatDesc.getDistributedBy(); }
@Override public List<TblColRef> getFactColumns() { return flatDesc.getFactColumns(); }
@Override public boolean useAlias() { return flatDesc.useAlias(); }
@Override protected AbstractExecutable createSqoopToFlatHiveStep(String jobWorkingDir, String cubeName) { KylinConfig config = flatDesc.getDataModel().getConfig(); PartitionDesc partitionDesc = flatDesc.getDataModel().getPartitionDesc(); String partCol = null; selectSql = escapeQuotationInSql(dataSource.convertSql(selectSql)); String hiveTable = flatDesc.getTableName(); String sqoopHome = config.getSqoopHome(); String filedDelimiter = config.getJdbcSourceFieldDelimiter(); bquery = dataSource.convertSql(bquery); if (partitionDesc.isPartitioned()) { SegmentRange segRange = flatDesc.getSegRange(); if (segRange != null && !segRange.isInfinite()) { if (partitionDesc.getPartitionDateColumnRef().getTableAlias().equals(splitTableAlias) String quotedPartCond = FlatTableSqlQuoteUtils.quoteIdentifierInSqlExpr(flatDesc, partitionDesc.getPartitionConditionBuilder().buildDateRangeCondition(partitionDesc, flatDesc.getSegment(), segRange), "`"); bquery += " WHERE " + quotedPartCond;
public static String generateRedistributeFlatTableStatement(IJoinedFlatTableDesc flatDesc, CubeDesc cubeDesc) { final String tableName = flatDesc.getTableName(); StringBuilder sql = new StringBuilder(); sql.append("INSERT OVERWRITE TABLE " + quoteIdentifier(tableName) + " SELECT * FROM " + quoteIdentifier(tableName)); if (flatDesc.getClusterBy() != null) { appendClusterStatement(sql, flatDesc.getClusterBy()); } else if (flatDesc.getDistributedBy() != null) { appendDistributeStatement(sql, Lists.newArrayList(flatDesc.getDistributedBy())); } else { int redistColumnCount = cubeDesc.getConfig().getHiveRedistributeColumnCount(); RowKeyColDesc[] rowKeyColDescs = cubeDesc.getRowkey().getRowKeyColumns(); if (rowKeyColDescs.length < redistColumnCount) redistColumnCount = rowKeyColDescs.length; List<TblColRef> redistColumns = Lists.newArrayListWithCapacity(redistColumnCount); for (int i = 0; i < redistColumnCount; i++) { redistColumns.add(rowKeyColDescs[i].getColRef()); } appendDistributeStatement(sql, redistColumns); } return sql.toString(); }
protected KylinConfig getConfig() { return flatDesc.getDataModel().getConfig(); }