public static Schema getTableSchema(@Nonnull ZkHelixPropertyStore<ZNRecord> propertyStore, @Nonnull String tableName) { String rawTableName = TableNameBuilder.extractRawTableName(tableName); Schema schema = getSchema(propertyStore, rawTableName); if (schema != null) { CommonConstants.Helix.TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableName); schema = getSchema(propertyStore, TableNameBuilder.OFFLINE.tableNameWithType(tableName)); schema.getSchemaName(), TableNameBuilder.extractRawTableName(tableName));
private TableConfig(@Nonnull String tableName, @Nonnull TableType tableType, @Nonnull SegmentsValidationAndRetentionConfig validationConfig, @Nonnull TenantConfig tenantConfig, @Nonnull IndexingConfig indexingConfig, @Nonnull TableCustomConfig customConfig, @Nullable QuotaConfig quotaConfig, @Nullable TableTaskConfig taskConfig, @Nullable RoutingConfig routingConfig) { _tableName = TableNameBuilder.forType(tableType).tableNameWithType(tableName); _tableType = tableType; _validationConfig = validationConfig; _tenantConfig = tenantConfig; _indexingConfig = indexingConfig; _customConfig = customConfig; _quotaConfig = quotaConfig; _taskConfig = taskConfig; _routingConfig = routingConfig; }
/** * Get all Pinot raw table names. * * @return Set of Pinot raw table names */ @Nonnull public List<String> getAllRawTables() { Set<String> rawTableNames = new HashSet<>(); for (String resourceName : getAllResources()) { if (TableNameBuilder.isTableResource(resourceName)) { rawTableNames.add(TableNameBuilder.extractRawTableName(resourceName)); } } return new ArrayList<>(rawTableNames); }
/** * Acquire a token from rate limiter based on the table name. * @param tableName original table name which could be raw. * @return true if there is no query quota specified for the table or a token can be acquired, otherwise return false. */ public boolean acquire(String tableName) { LOGGER.debug("Trying to acquire token for table: {}", tableName); String offlineTableName = null; String realtimeTableName = null; QueryQuotaConfig offlineTableQueryQuotaConfig = null; QueryQuotaConfig realtimeTableQueryQuotaConfig = null; CommonConstants.Helix.TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableName); if (tableType == CommonConstants.Helix.TableType.OFFLINE) { offlineTableName = tableName; offlineTableQueryQuotaConfig = _rateLimiterMap.get(tableName); } else if (tableType == CommonConstants.Helix.TableType.REALTIME) { realtimeTableName = tableName; realtimeTableQueryQuotaConfig = _rateLimiterMap.get(tableName); } else { offlineTableName = TableNameBuilder.OFFLINE.tableNameWithType(tableName); realtimeTableName = TableNameBuilder.REALTIME.tableNameWithType(tableName); offlineTableQueryQuotaConfig = _rateLimiterMap.get(offlineTableName); realtimeTableQueryQuotaConfig = _rateLimiterMap.get(realtimeTableName); } boolean offlineQuotaOk = offlineTableQueryQuotaConfig == null || tryAcquireToken(offlineTableName, offlineTableQueryQuotaConfig); boolean realtimeQuotaOk = realtimeTableQueryQuotaConfig == null || tryAcquireToken(realtimeTableName, realtimeTableQueryQuotaConfig); return offlineQuotaOk && realtimeQuotaOk; }
public String getGroupId(String resourceName) { return _groupIdMap.get(TableNameBuilder.REALTIME.tableNameWithType(resourceName)); }
@Override public Map<String, ?> unapply(Map<String, ?> childKeys, String pathPrefix) { String tableNameWithSuffix = childKeys.filterKeys(key -> key.startsWith("table.name")).head()._2.toString(); return ((Map<String, Object>) childKeys).filterKeys(key -> !key.startsWith("table.name")) .put("table.name", TableNameBuilder.extractRawTableName(tableNameWithSuffix)); } }
@Override protected void processTable(String tableNameWithType) { CommonConstants.Helix.TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableNameWithType); if (tableType == CommonConstants.Helix.TableType.REALTIME) { runRelocation(tableNameWithType); } }
String tableNameWithType = entry.getKey(); QueryQuotaConfig queryQuotaConfig = entry.getValue(); String rawTableName = TableNameBuilder.extractRawTableName(tableNameWithType); TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableNameWithType);
throws Exception { final TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableName); if (!tableType.equals(TableType.OFFLINE)) { String serverTenant = TableNameBuilder.forType(tableType).tableNameWithType(tenantName); List<String> instancesInClusterWithTag = HelixHelper.getInstancesWithTag(helixManager, serverTenant); List<String> enabledInstancesWithTag = HelixHelper.getEnabledInstancesWithTag(helixManager, serverTenant);
private List<String> getAllTables() { List<String> tableNames = new ArrayList<>(); List<String> resources = helixAdmin.getResourcesInCluster(clusterName); for (String resourceName : resources) { if (TableNameBuilder.isTableResource(resourceName)) { tableNames.add(resourceName); } } return tableNames; } }
/** * Return whether the given resource name represents a table resource. * * @param resourceName Resource name * @return Whether the resource name represents a table resource */ public static boolean isTableResource(@Nonnull String resourceName) { return OFFLINE.tableHasTypeSuffix(resourceName) || REALTIME.tableHasTypeSuffix(resourceName); } }
public void setPartition(String resourceName, String partition) { _partitionMap.put(TableNameBuilder.REALTIME.tableNameWithType(resourceName), partition); }
public IdealStateBuilderUtil(String tableNameWithType) { _tableName = tableNameWithType; _rawTableName = TableNameBuilder.extractRawTableName(tableNameWithType); _idealState = new IdealState(tableNameWithType); }
@GET @Produces(MediaType.APPLICATION_JSON) @Path("/timeBoundary/{tableName}") @ApiOperation(value = "Debugging time boundary service") @ApiResponses(value = {@ApiResponse(code = 200, message = "Time boundary information of a table"), @ApiResponse(code = 500, message = "Internal server error")}) public String debugTimeBoundaryService( @ApiParam(value = "Name of the table") @PathParam("tableName") String tableName) { try { String response = "{}"; if (!tableName.isEmpty()) { CommonConstants.Helix.TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableName); if (tableType == null) { tableName = TableNameBuilder.OFFLINE.tableNameWithType(tableName); } TimeBoundaryService.TimeBoundaryInfo tbInfo = _timeBoundaryService.getTimeBoundaryInfoFor(tableName); if (tbInfo != null) { response = tbInfo.toJsonString(); } } return response; } catch (Exception e) { LOGGER.error("Caught exception while processing GET request", e); _brokerMetrics.addMeteredGlobalValue(BrokerMeter.UNCAUGHT_GET_EXCEPTIONS, 1); throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR); } } }
@Override protected void processTable(String tableNameWithType) { CommonConstants.Helix.TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableNameWithType); if (tableType == CommonConstants.Helix.TableType.OFFLINE) { TableConfig tableConfig = _pinotHelixResourceManager.getTableConfig(tableNameWithType); if (tableConfig == null) { LOGGER.warn("Failed to find table config for table: {}, skipping validation", tableNameWithType); return; } validateOfflineSegmentPush(tableConfig); } }
@Override public Boolean executeTask(@Nonnull PinotTaskConfig pinotTaskConfig) { assertTrue(MINION_CONTEXT.getDataDir().exists()); assertNotNull(MINION_CONTEXT.getMinionMetrics()); assertNotNull(MINION_CONTEXT.getMinionVersion()); assertEquals(pinotTaskConfig.getTaskType(), TestTaskGenerator.TASK_TYPE); Map<String, String> configs = pinotTaskConfig.getConfigs(); assertEquals(configs.size(), 2); String offlineTableName = configs.get("tableName"); assertEquals(TableNameBuilder.getTableTypeFromTableName(offlineTableName), TableType.OFFLINE); String rawTableName = TableNameBuilder.extractRawTableName(offlineTableName); assertTrue(rawTableName.equals(TABLE_NAME_1) || rawTableName.equals(TABLE_NAME_2)); assertEquals(configs.get("tableType"), TableType.OFFLINE.toString()); do { if (_cancelled) { throw new TaskCancelledException("Task has been cancelled"); } } while (HOLD.get()); return true; } };
/** * Get all Pinot table names (server resources). * * @return List of Pinot table names */ @Nonnull public List<String> getAllTables() { List<String> tableNames = new ArrayList<>(); for (String resourceName : getAllResources()) { if (TableNameBuilder.isTableResource(resourceName)) { tableNames.add(resourceName); } } return tableNames; }
/** * Get the table type based on the given table name with type suffix. * * @param tableName Table name with or without type suffix * @return Table type for the given table name, null if cannot be determined by table name */ @Nullable public static TableType getTableTypeFromTableName(@Nonnull String tableName) { if (OFFLINE.tableHasTypeSuffix(tableName)) { return TableType.OFFLINE; } if (REALTIME.tableHasTypeSuffix(tableName)) { return TableType.REALTIME; } return null; }
CommonConstants.Helix.TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableNameWithType); TableNameBuilder.REALTIME.tableNameWithType(TableNameBuilder.extractRawTableName(tableNameWithType)); if (_routingTableBuilderMap.containsKey(realtimeTableName)) { tableForTimeBoundaryUpdate = tableNameWithType; TableNameBuilder.OFFLINE.tableNameWithType(TableNameBuilder.extractRawTableName(tableNameWithType)); if (_routingTableBuilderMap.containsKey(offlineTableName)) {
private String getTableNameWithType(@Nonnull String tableNameOptType, @Nullable CommonConstants.Helix.TableType tableType) { if (tableType != null) { if (tableType == CommonConstants.Helix.TableType.OFFLINE) { return TableNameBuilder.OFFLINE.tableNameWithType(tableNameOptType); } else { return TableNameBuilder.REALTIME.tableNameWithType(tableNameOptType); } } else { return tableNameOptType; } } }