private static FeedPolicyEntity initializeBasicPolicy() { Map<String, String> policyParams = new HashMap<>(); policyParams.put(FeedPolicyAccessor.ELASTIC, Boolean.toString(false)); String description = "Basic"; return new FeedPolicyEntity(MetadataConstants.METADATA_DATAVERSE_NAME, "Basic", description, policyParams); }
public Object dropFeedPolicy(FeedPolicyEntity feedPolicy) { synchronized (feedPolicies) { Map<String, FeedPolicyEntity> p = feedPolicies.get(feedPolicy.getDataverseName()); if (p != null && p.get(feedPolicy.getPolicyName()) != null) { return p.remove(feedPolicy).getPolicyName(); } return null; } }
aString.setValue(feedPolicy.getDataverseName()); stringSerde.serialize(aString, tupleBuilder.getDataOutput()); tupleBuilder.addFieldEndOffset(); aString.setValue(feedPolicy.getPolicyName()); stringSerde.serialize(aString, tupleBuilder.getDataOutput()); tupleBuilder.addFieldEndOffset(); aString.setValue(feedPolicy.getDataverseName()); stringSerde.serialize(aString, fieldValue.getDataOutput()); recordBuilder.addField(MetadataRecordTypes.FEED_POLICY_ARECORD_DATAVERSE_NAME_FIELD_INDEX, fieldValue); aString.setValue(feedPolicy.getPolicyName()); stringSerde.serialize(aString, fieldValue.getDataOutput()); recordBuilder.addField(MetadataRecordTypes.FEED_POLICY_ARECORD_POLICY_NAME_FIELD_INDEX, fieldValue); aString.setValue(feedPolicy.getDescription()); stringSerde.serialize(aString, fieldValue.getDataOutput()); recordBuilder.addField(MetadataRecordTypes.FEED_POLICY_ARECORD_DESCRIPTION_FIELD_INDEX, fieldValue); Map<String, String> properties = feedPolicy.getProperties(); UnorderedListBuilder listBuilder = new UnorderedListBuilder(); listBuilder.reset((AUnorderedListType) MetadataRecordTypes.FEED_POLICY_RECORDTYPE
throw new AlgebricksException("Feed not configured with a policy"); feedPolicy.getProperties().put(BuiltinFeedPolicies.CONFIG_FEED_POLICY_KEY, feedPolicy.getPolicyName()); FeedConnectionId feedConnectionId = new FeedConnectionId(getId().getDataverseName(), getId().getDatasourceName(), getTargetDataset()); FeedCollectOperatorDescriptor feedCollector = new FeedCollectOperatorDescriptor(jobSpec, feedConnectionId, feedOutputType, feedDesc, feedPolicy.getProperties(), getLocation());
Map<String, String> policyProperties = sourceFeedPolicy.getProperties(); policyProperties.putAll(cfps.getProperties()); newPolicy = new FeedPolicyEntity(dataverse, policy, description, policyProperties); } else { Properties prop = new Properties(); newPolicy = new FeedPolicyEntity(dataverse, policy, description, policyProperties);
public static FeedPolicyEntity getFeedPolicy(String policyName) { for (FeedPolicyEntity policy : POLICIES) { if (policy.getPolicyName().equalsIgnoreCase(policyName)) { return policy; } } return null; }
&& ((LSMTreeInsertDeleteOperatorDescriptor) opDesc).isPrimary()) { metaOp = new FeedMetaOperatorDescriptor(jobSpec, feedConnectionId, opDesc, feedPolicyEntity.getProperties(), FeedRuntimeType.STORE); opId = metaOp.getOperatorId(); opDesc.setOperatorId(opId); feedPolicyEntity.getProperties(), FeedRuntimeType.COMPUTE); opId = metaOp.getOperatorId(); opDesc.setOperatorId(opId);
public ConnectFeedStatement(Pair<Identifier, Identifier> feedNameCmp, Pair<Identifier, Identifier> datasetNameCmp, List<FunctionSignature> appliedFunctions, String policy, String whereClauseBody, int varCounter) { if (feedNameCmp.first != null && datasetNameCmp.first != null && !feedNameCmp.first.getValue().equals(datasetNameCmp.first.getValue())) { throw new IllegalArgumentException("Dataverse for source feed and target dataset do not match"); } this.dataverseName = feedNameCmp.first != null ? feedNameCmp.first : datasetNameCmp.first != null ? datasetNameCmp.first : null; this.datasetName = datasetNameCmp.second; this.feedName = feedNameCmp.second.getValue(); this.policy = policy != null ? policy : BuiltinFeedPolicies.DEFAULT_POLICY.getPolicyName(); this.whereClauseBody = whereClauseBody; this.varCounter = varCounter; this.appliedFunctions = appliedFunctions; }
private static FeedPolicyEntity initializeAdvancedFTSpillPolicy() { Map<String, String> policyParams = new HashMap<>(); policyParams.put(FeedPolicyAccessor.ELASTIC, Boolean.toString(false)); policyParams.put(FeedPolicyAccessor.FLOWCONTROL_ENABLED, Boolean.toString(true)); policyParams.put(FeedPolicyAccessor.SPILL_TO_DISK_ON_CONGESTION, "" + Boolean.TRUE); policyParams.put(FeedPolicyAccessor.MAX_SPILL_SIZE_ON_DISK, "" + Long.toString(FeedPolicyAccessor.NO_LIMIT)); String description = "FlowControl 100% Spill during congestion"; return new FeedPolicyEntity(MetadataConstants.METADATA_DATAVERSE_NAME, "Spill", description, policyParams); }
public Object addFeedPolicyIfNotExists(FeedPolicyEntity feedPolicy) { synchronized (feedPolicy) { Map<String, FeedPolicyEntity> p = feedPolicies.get(feedPolicy.getDataverseName()); if (p == null) { p = new HashMap<>(); p.put(feedPolicy.getPolicyName(), feedPolicy); feedPolicies.put(feedPolicy.getDataverseName(), p); } else { if (p.get(feedPolicy.getPolicyName()) == null) { p.put(feedPolicy.getPolicyName(), feedPolicy); } } return null; } }
@Override public void addFeedPolicy(TxnId txnId, FeedPolicyEntity feedPolicy) throws AlgebricksException, RemoteException { try { // Insert into the 'FeedPolicy' dataset. FeedPolicyTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getFeedPolicyTupleTranslator(true); ITupleReference feedPolicyTuple = tupleReaderWriter.getTupleFromMetadataEntity(feedPolicy); insertTupleIntoIndex(txnId, MetadataPrimaryIndexes.FEED_POLICY_DATASET, feedPolicyTuple); } catch (HyracksDataException e) { if (e.getComponent().equals(ErrorCode.HYRACKS) && e.getErrorCode() == ErrorCode.DUPLICATE_KEY) { throw new AlgebricksException("A feed policy with this name " + feedPolicy.getPolicyName() + " already exists in dataverse '" + feedPolicy.getPolicyName() + "'.", e); } else { throw new AlgebricksException(e); } } }
private static FeedPolicyEntity initializeAdvancedFTElasticPolicy() { Map<String, String> policyParams = new HashMap<>(); policyParams.put(FeedPolicyAccessor.ELASTIC, Boolean.toString(true)); policyParams.put(FeedPolicyAccessor.FLOWCONTROL_ENABLED, Boolean.toString(true)); policyParams.put(FeedPolicyAccessor.LOGGING_STATISTICS, Boolean.toString(true)); String description = "Basic Monitored Fault-Tolerant Elastic"; return new FeedPolicyEntity(MetadataConstants.METADATA_DATAVERSE_NAME, "AdvancedFT_Elastic", description, policyParams); }
dropFeedPolicy(txnId, dataverseName, feedPolicy.getPolicyName());
private static FeedPolicyEntity initializeAdvancedFTDiscardPolicy() { Map<String, String> policyParams = new HashMap<>(); policyParams.put(FeedPolicyAccessor.ELASTIC, Boolean.toString(false)); policyParams.put(FeedPolicyAccessor.FLOWCONTROL_ENABLED, Boolean.toString(true)); policyParams.put(FeedPolicyAccessor.MAX_SPILL_SIZE_ON_DISK, Boolean.toString(false)); policyParams.put(FeedPolicyAccessor.MAX_FRACTION_DISCARD, "100"); policyParams.put(FeedPolicyAccessor.LOGGING_STATISTICS, Boolean.toString(true)); String description = "FlowControl 100% Discard during congestion"; return new FeedPolicyEntity(MetadataConstants.METADATA_DATAVERSE_NAME, "Discard", description, policyParams); }
private FeedPolicyEntity createFeedPolicyFromARecord(ARecord feedPolicyRecord) { FeedPolicyEntity feedPolicy = null; String dataverseName = ((AString) feedPolicyRecord .getValueByPos(MetadataRecordTypes.FEED_POLICY_ARECORD_DATAVERSE_NAME_FIELD_INDEX)).getStringValue(); String policyName = ((AString) feedPolicyRecord .getValueByPos(MetadataRecordTypes.FEED_POLICY_ARECORD_POLICY_NAME_FIELD_INDEX)).getStringValue(); String description = ((AString) feedPolicyRecord .getValueByPos(MetadataRecordTypes.FEED_POLICY_ARECORD_DESCRIPTION_FIELD_INDEX)).getStringValue(); IACursor cursor = ((AUnorderedList) feedPolicyRecord .getValueByPos(MetadataRecordTypes.FEED_POLICY_ARECORD_PROPERTIES_FIELD_INDEX)).getCursor(); Map<String, String> policyParamters = new HashMap<>(); String key; String value; while (cursor.next()) { ARecord field = (ARecord) cursor.get(); key = ((AString) field.getValueByPos(MetadataRecordTypes.PROPERTIES_NAME_FIELD_INDEX)).getStringValue(); value = ((AString) field.getValueByPos(MetadataRecordTypes.PROPERTIES_VALUE_FIELD_INDEX)).getStringValue(); policyParamters.put(key, value); } feedPolicy = new FeedPolicyEntity(dataverseName, policyName, description, policyParamters); return feedPolicy; }