/** * Get all Pinot raw table names. * * @return Set of Pinot raw table names */ @Nonnull public List<String> getAllRawTables() { Set<String> rawTableNames = new HashSet<>(); for (String resourceName : getAllResources()) { if (TableNameBuilder.isTableResource(resourceName)) { rawTableNames.add(TableNameBuilder.extractRawTableName(resourceName)); } } return new ArrayList<>(rawTableNames); }
@Override public Map<String, ?> unapply(Map<String, ?> childKeys, String pathPrefix) { String tableNameWithSuffix = childKeys.filterKeys(key -> key.startsWith("table.name")).head()._2.toString(); return ((Map<String, Object>) childKeys).filterKeys(key -> !key.startsWith("table.name")) .put("table.name", TableNameBuilder.extractRawTableName(tableNameWithSuffix)); } }
public IdealStateBuilderUtil(String tableNameWithType) { _tableName = tableNameWithType; _rawTableName = TableNameBuilder.extractRawTableName(tableNameWithType); _idealState = new IdealState(tableNameWithType); }
protected void removeSegmentFromStore(String tableNameWithType, String segmentId) { final String rawTableName = TableNameBuilder.extractRawTableName(tableNameWithType); if (_dataDir != null) { URI fileToMoveURI;
String rawTableName = TableNameBuilder.extractRawTableName(tableName); int nextSeqNum = STARTING_SEQUENCE_NUMBER;
public static Schema getTableSchema(@Nonnull ZkHelixPropertyStore<ZNRecord> propertyStore, @Nonnull String tableName) { String rawTableName = TableNameBuilder.extractRawTableName(tableName); Schema schema = getSchema(propertyStore, rawTableName); if (schema != null) { schema.getSchemaName(), TableNameBuilder.extractRawTableName(tableName));
String tableNameWithType = entry.getKey(); QueryQuotaConfig queryQuotaConfig = entry.getValue(); String rawTableName = TableNameBuilder.extractRawTableName(tableNameWithType); TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableNameWithType);
return QueryException.getException(QueryException.PQL_PARSING_ERROR, e).toString(); String tableName = TableNameBuilder.extractRawTableName(brokerRequest.getQuerySource().getTableName());
@Override public Boolean executeTask(@Nonnull PinotTaskConfig pinotTaskConfig) { assertTrue(MINION_CONTEXT.getDataDir().exists()); assertNotNull(MINION_CONTEXT.getMinionMetrics()); assertNotNull(MINION_CONTEXT.getMinionVersion()); assertEquals(pinotTaskConfig.getTaskType(), TestTaskGenerator.TASK_TYPE); Map<String, String> configs = pinotTaskConfig.getConfigs(); assertEquals(configs.size(), 2); String offlineTableName = configs.get("tableName"); assertEquals(TableNameBuilder.getTableTypeFromTableName(offlineTableName), TableType.OFFLINE); String rawTableName = TableNameBuilder.extractRawTableName(offlineTableName); assertTrue(rawTableName.equals(TABLE_NAME_1) || rawTableName.equals(TABLE_NAME_2)); assertEquals(configs.get("tableType"), TableType.OFFLINE.toString()); do { if (_cancelled) { throw new TaskCancelledException("Task has been cancelled"); } } while (HOLD.get()); return true; } };
String rawTableName = TableNameBuilder.extractRawTableName(newTableConfig.getTableName()); LOGGER.info("Validating table configs for Table: {}", rawTableName);
assert offlineBrokerRequest != null || realtimeBrokerRequest != null; String rawTableName = TableNameBuilder.extractRawTableName(originalBrokerRequest.getQuerySource().getTableName()); long scatterGatherStartTimeNs = System.nanoTime(); AsyncQueryResponse asyncQueryResponse = _queryRouter
String rawTableName = TableNameBuilder.extractRawTableName(tableNameWithType); LOGGER.info("Initializing rate limiter for table {}", tableNameWithType);
@Override protected SegmentConversionResult convert(@Nonnull PinotTaskConfig pinotTaskConfig, @Nonnull File originalIndexDir, @Nonnull File workingDir) throws Exception { Map<String, String> configs = pinotTaskConfig.getConfigs(); String tableNameWithType = configs.get(MinionConstants.TABLE_NAME_KEY); String rawTableName = TableNameBuilder.extractRawTableName(tableNameWithType); SegmentPurger.RecordPurgerFactory recordPurgerFactory = MINION_CONTEXT.getRecordPurgerFactory(); SegmentPurger.RecordPurger recordPurger = recordPurgerFactory != null ? recordPurgerFactory.getRecordPurger(rawTableName) : null; SegmentPurger.RecordModifierFactory recordModifierFactory = MINION_CONTEXT.getRecordModifierFactory(); SegmentPurger.RecordModifier recordModifier = recordModifierFactory != null ? recordModifierFactory.getRecordModifier(rawTableName) : null; SegmentPurger segmentPurger = new SegmentPurger(originalIndexDir, workingDir, recordPurger, recordModifier); File purgedSegmentFile = segmentPurger.purgeSegment(); if (purgedSegmentFile == null) { purgedSegmentFile = originalIndexDir; } return new SegmentConversionResult.Builder().setFile(purgedSegmentFile).setTableNameWithType(tableNameWithType) .setSegmentName(configs.get(MinionConstants.SEGMENT_NAME_KEY)) .setCustomProperty(RECORD_PURGER_KEY, segmentPurger.getRecordPurger()) .setCustomProperty(RECORD_MODIFIER_KEY, segmentPurger.getRecordModifier()) .setCustomProperty(NUM_RECORDS_PURGED_KEY, segmentPurger.getNumRecordsPurged()) .setCustomProperty(NUM_RECORDS_MODIFIED_KEY, segmentPurger.getNumRecordsModified()).build(); }
final String rawTableName = TableNameBuilder.extractRawTableName(tableName); final int nInstances = 6; final int nPartitions = 16;
Assert.assertEquals(llcSegmentName.getPartitionId(), partition); Assert.assertEquals(llcSegmentName.getTableName(), TableNameBuilder.extractRawTableName(tableConfig.getTableName())); Assert.assertEquals(metadata.getNumReplicas(), nReplicas);
private void advanceASeqForPartition(IdealState idealState, FakePinotLLCRealtimeSegmentManager segmentManager, PartitionAssignment partitionAssignment, String segmentName, int partition, int nextSeqNum, long nextOffset, TableConfig tableConfig) { String tableName = tableConfig.getTableName(); String rawTableName = TableNameBuilder.extractRawTableName(tableName); LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); segmentManager.updateOldSegmentMetadataZNRecord(tableName, llcSegmentName, nextOffset); LLCSegmentName newLlcSegmentName = new LLCSegmentName(rawTableName, partition, nextSeqNum, System.currentTimeMillis()); CommittingSegmentDescriptor committingSegmentDescriptor = new CommittingSegmentDescriptor(segmentName, nextOffset, 0); segmentManager.createNewSegmentMetadataZNRecord(tableConfig, llcSegmentName, newLlcSegmentName, partitionAssignment, committingSegmentDescriptor, false); segmentManager.updateIdealStateOnSegmentCompletion(idealState, segmentName, newLlcSegmentName.getSegmentName(), partitionAssignment); }
String rawTableName = TableNameBuilder.extractRawTableName(realtimeTableName);
@Test public void testCommitSegmentWhenControllerWentThroughGC() throws InvalidConfigException { FakePinotLLCRealtimeSegmentManager segmentManager1 = new FakePinotLLCRealtimeSegmentManager(null); FakePinotLLCRealtimeSegmentManager segmentManager2 = new FakePinotLLCRealtimeSegmentManagerII(null, FakePinotLLCRealtimeSegmentManagerII.SCENARIO_1_ZK_VERSION_NUM_HAS_CHANGE); FakePinotLLCRealtimeSegmentManager segmentManager3 = new FakePinotLLCRealtimeSegmentManagerII(null, FakePinotLLCRealtimeSegmentManagerII.SCENARIO_2_METADATA_STATUS_HAS_CHANGE); final String rtTableName = "table_REALTIME"; final String rawTableName = TableNameBuilder.extractRawTableName(rtTableName); setupSegmentManager(segmentManager1, rtTableName); setupSegmentManager(segmentManager2, rtTableName); setupSegmentManager(segmentManager3, rtTableName); // Now commit the first segment of partition 6. final int committingPartition = 6; final long nextOffset = 3425666L; SegmentCompletionProtocol.Request.Params reqParams = new SegmentCompletionProtocol.Request.Params(); LLCRealtimeSegmentZKMetadata committingSegmentMetadata = new LLCRealtimeSegmentZKMetadata(segmentManager2._records.get(committingPartition)); reqParams.withSegmentName(committingSegmentMetadata.getSegmentName()).withOffset(nextOffset); CommittingSegmentDescriptor committingSegmentDescriptor = CommittingSegmentDescriptor.fromSegmentCompletionReqParams(reqParams); boolean status = segmentManager1.commitSegmentMetadata(rawTableName, committingSegmentDescriptor); Assert.assertTrue(status); // Committing segment metadata succeeded. status = segmentManager2.commitSegmentMetadata(rawTableName, committingSegmentDescriptor); Assert.assertFalse(status); // Committing segment metadata failed. status = segmentManager3.commitSegmentMetadata(rawTableName, committingSegmentDescriptor); Assert.assertFalse(status); // Committing segment metadata failed. }
String rawTableName = TableNameBuilder.extractRawTableName(realtimeTableName); committingSegmentMetadata.setDownloadUrl( ControllerConf.constructDownloadUrl(rawTableName, committingSegmentNameStr, _controllerConf.generateVipUrl()));
throws Exception { final String tableName = "myTable_REALTIME"; final String rawTableName = TableNameBuilder.extractRawTableName(tableName); List<String> allTableNames = new ArrayList<String>(); allTableNames.add(tableName);