@Override public void createIfAbsent(Kv.CreateTableInfo request, StreamObserver<Kv.CreateTableInfo> responseObserver) { LOGGER.info("Kv.createIfAbsent request received. request: {}", toStringUtils.toOneLineString(request)); grpcServerWrapper.wrapGrpcServerRunnable(responseObserver, () -> { StorageBasic.StorageLocator storageLocator = request.getStorageLocator(); List<Fragment> fragments = null; Dtable createResult = null; Dtable createTemplate = storageMetaClient.getTable(storageLocator.getNamespace(), storageLocator.getName()); // todo: add transaction control if (createTemplate == null) { createTemplate = typeConversionUtils.toDtable(request); createResult = storageMetaClient.createTable(createTemplate); if (createResult != null) { fragments = storageMetaClient.createFragmentsForTable(createResult); } } else { fragments = storageMetaClient.getFragmentsByTableId(createTemplate.getTableId()); createResult = createTemplate; } Kv.CreateTableInfo result = null; // todo: add more result check if (!fragments.isEmpty()) { // createResult = storageMetaClient.createTable(createTemplate); result = typeConversionUtils.toCreateTableInfo(createResult); } responseObserver.onNext(result); responseObserver.onCompleted(); }); }
public synchronized void loadProxy() { LOGGER.info("[FEDERATION][PROXY][SELECTION] loading"); storageMetaClient.init(federationServerUtils.getMetaServiceEndpoint()); proxies.clear(); Node node = new Node(); node.setType(NodeType.PROXY.name()); node.setStatus(NodeStatus.HEALTHY.name()); List<Node> nodes = storageMetaClient.getNodes(node); BasicMeta.Endpoint endpoint = null; for (Node cur : nodes) { endpoint = typeConversionUtils.toEndpoint(cur); proxies.add(endpoint); } }
@Override public void onProcess() { // LOGGER.info("DtableRecvConsumeAction.onProcess"); super.onProcess(); List<ByteString> rawMaps = Lists.newArrayList(); transferBroker.drainTo(rawMaps); try { if (!rawMaps.isEmpty()) { for (ByteString rawMapBS : rawMaps) { DataStructure.RawMap rawMap = DataStructure.RawMap.parseFrom(rawMapBS); for (DataStructure.RawEntry entry : rawMap.getEntriesList()) { Kv.Operand operand = typeConversionUtils.toOperand(entry); // LOGGER.info("operand: {}", toStringUtils.toOneLineString(operand)); operandBroker.put(operand); ++entryCount; } } } } catch (Exception e) { LOGGER.error(errorUtils.getStackTrace(e)); throw new RuntimeException(e); } }
public Dtable toDtable(Kv.CreateTableInfo createTableInfo) { Dtable result = toDtable(createTableInfo.getStorageLocator()); result.setTotalFragments(createTableInfo.getFragmentCount()); result.setStatus(DtableStatus.NORMAL.name()); return result; }
public Kv.CreateTableInfo toCreateTableInfo(Dtable dtable) { StorageBasic.StorageLocator storageLocator = toStorageLocator(dtable); synchronized (createTableInfoBuilderLock) { createTableInfoBuilder.clear(); createTableInfoBuilder.setStorageLocator(storageLocator); Integer fragments = dtable.getTotalFragments(); if (fragments != null) { createTableInfoBuilder.setFragmentCount(fragments); } return createTableInfoBuilder.build(); } }
Dtable dtable = storageMetaClient.getTable(typeConversionUtils.toDtable(finalStorageLocator)); int fragmentCount = finalStorageLocator.getFragment();
public synchronized BasicMeta.Endpoint getRollEndpoint() { if (rollEndpoint == null) { init(); try { Node node = new Node(); node.setType(NodeType.ROLL.name()); node.setStatus(NodeStatus.HEALTHY.name()); List<Node> nodes = storageMetaClient.getNodes(node); if (nodes == null || nodes.isEmpty()) { throw new IllegalStateException("no valid roll node exists"); } rollEndpoint = typeConversionUtils.toEndpoint(nodes.get(0)); } catch (Exception e) { LOGGER.info("[ROLL][INIT] init roll endpoint failed. using default"); rollEndpoint = RuntimeConstants.getLocalEndpoint(8011); } } return rollEndpoint; } }
public StorageBasic.StorageLocator join(Processor.BinaryProcess request, Node node) { GrpcAsyncClientContext<ProcessServiceGrpc.ProcessServiceStub, Processor.BinaryProcess, StorageBasic.StorageLocator> context = rollProcessorServiceCallModelFactory.createBinaryProcessToStorageLocatorContext(); DelayedResult<StorageBasic.StorageLocator> delayedResult = new SingleDelayedResult<>(); context.setLatchInitCount(1) .setEndpoint(typeConversionUtils.toEndpoint(node)) .setFinishTimeout(RuntimeConstants.DEFAULT_WAIT_TIME, RuntimeConstants.DEFAULT_TIMEUNIT) .setCalleeStreamingMethodInvoker(ProcessServiceGrpc.ProcessServiceStub::join) .setCallerStreamObserverClassAndArguments(EggProcessorUnaryProcessToStorageLocatorResponseObserver.class, delayedResult); GrpcStreamingClientTemplate<ProcessServiceGrpc.ProcessServiceStub, Processor.BinaryProcess, StorageBasic.StorageLocator> template = rollProcessorServiceCallModelFactory.createBinaryProcessToStorageLocatorTemplate(); template.setGrpcAsyncClientContext(context); StorageBasic.StorageLocator result = null; try { result = template.calleeStreamingRpcWithImmediateDelayedResult(request, delayedResult); } catch (InvocationTargetException e) { throw new RuntimeException(e); } return result; }
public Kv.Operand get(Kv.Operand request, StoreInfo storeInfo, Node node) { DelayedResult<Kv.Operand> delayedResult = new SingleDelayedResult<>(); GrpcAsyncClientContext<KVServiceGrpc.KVServiceStub, Kv.Operand, Kv.Operand> context = rollKvCallModelFactory.createOperandToOperandContext(); context.setLatchInitCount(1) .setEndpoint(typeConversionUtils.toEndpoint(node)) .setFinishTimeout(RuntimeConstants.DEFAULT_WAIT_TIME, RuntimeConstants.DEFAULT_TIMEUNIT) .setCalleeStreamingMethodInvoker(KVServiceGrpc.KVServiceStub::get) .setCallerStreamObserverClassAndArguments(StorageKvOperandToOperandObserver.class, delayedResult) .setGrpcMetadata(MetaConstants.createMetadataFromStoreInfo(storeInfo)); Kv.Operand result = null; GrpcStreamingClientTemplate<KVServiceGrpc.KVServiceStub, Kv.Operand, Kv.Operand> template = rollKvCallModelFactory.createOperandToOperandTemplate(); template.setGrpcAsyncClientContext(context); try { result = template.calleeStreamingRpcWithImmediateDelayedResult(request, delayedResult); } catch (InvocationTargetException e) { LOGGER.error(errorUtils.getStackTrace(e)); throw new RuntimeException(e); } return result; }
public Kv.Count count(Kv.Empty request, StoreInfo storeInfo, Node node) { DelayedResult<Kv.Count> delayedResult = new SingleDelayedResult<>(); GrpcAsyncClientContext<KVServiceGrpc.KVServiceStub, Kv.Empty, Kv.Count> context = rollKvCallModelFactory.createEmptyToCountContext(); context.setLatchInitCount(1) .setEndpoint(typeConversionUtils.toEndpoint(node)) .setFinishTimeout(RuntimeConstants.DEFAULT_WAIT_TIME, RuntimeConstants.DEFAULT_TIMEUNIT) .setCalleeStreamingMethodInvoker(KVServiceGrpc.KVServiceStub::count) .setCallerStreamObserverClassAndArguments(StorageKvCountResponseObserver.class, delayedResult) .setGrpcMetadata(MetaConstants.createMetadataFromStoreInfo(storeInfo)); GrpcStreamingClientTemplate<KVServiceGrpc.KVServiceStub, Kv.Empty, Kv.Count> template = rollKvCallModelFactory.createEmptyToCountTemplate(); template.setGrpcAsyncClientContext(context); Kv.Count result = null; try { result = template.calleeStreamingRpcWithImmediateDelayedResult(request, delayedResult); } catch (InvocationTargetException e) { LOGGER.error(errorUtils.getStackTrace(e)); throw new RuntimeException(e); } return result; } }
.setEndpoint(typeConversionUtils.toEndpoint(node)) .setFinishTimeout(RuntimeConstants.DEFAULT_WAIT_TIME, RuntimeConstants.DEFAULT_TIMEUNIT) .setCalleeStreamingMethodInvoker(calleeStreamingStubMethodInvoker)
public OperandBroker reduce(Processor.UnaryProcess request, Node node) { OperandBroker result = rollModelFactory.createOperandBroker(); GrpcAsyncClientContext<ProcessServiceGrpc.ProcessServiceStub, Processor.UnaryProcess, Kv.Operand> context = rollProcessorServiceCallModelFactory.createUnaryProcessToOperandContext(); context.setLatchInitCount(1) .setEndpoint(typeConversionUtils.toEndpoint(node)) .setFinishTimeout(RuntimeConstants.DEFAULT_WAIT_TIME, RuntimeConstants.DEFAULT_TIMEUNIT) .setCalleeStreamingMethodInvoker(ProcessServiceGrpc.ProcessServiceStub::reduce) .setCallerStreamObserverClassAndArguments(EggProcessorReduceResponseStreamObserver.class, result); GrpcStreamingClientTemplate<ProcessServiceGrpc.ProcessServiceStub, Processor.UnaryProcess, Kv.Operand> template = rollProcessorServiceCallModelFactory.createUnaryProcessToOperandTemplate(); template.setGrpcAsyncClientContext(context); template.calleeStreamingRpc(request); return result; }
public Kv.Operand putIfAbsent(Kv.Operand request, StoreInfo storeInfo, Node node) { DelayedResult<Kv.Operand> delayedResult = new SingleDelayedResult<>(); GrpcAsyncClientContext<KVServiceGrpc.KVServiceStub, Kv.Operand, Kv.Operand> context = rollKvCallModelFactory.createOperandToOperandContext(); context.setLatchInitCount(1) .setEndpoint(typeConversionUtils.toEndpoint(node)) .setFinishTimeout(RuntimeConstants.DEFAULT_WAIT_TIME, RuntimeConstants.DEFAULT_TIMEUNIT) .setCalleeStreamingMethodInvoker(KVServiceGrpc.KVServiceStub::putIfAbsent) .setCallerStreamObserverClassAndArguments(StorageKvOperandToOperandObserver.class, delayedResult) .setGrpcMetadata(MetaConstants.createMetadataFromStoreInfo(storeInfo)); Kv.Operand result = null; GrpcStreamingClientTemplate<KVServiceGrpc.KVServiceStub, Kv.Operand, Kv.Operand> template = rollKvCallModelFactory.createOperandToOperandTemplate(); template.setGrpcAsyncClientContext(context); try { result = template.calleeStreamingRpcWithImmediateDelayedResult(request, delayedResult); } catch (InvocationTargetException e) { LOGGER.error(errorUtils.getStackTrace(e)); } return result; }
public Kv.Operand delete(Kv.Operand request, StoreInfo storeInfo, Node node) { DelayedResult<Kv.Operand> delayedResult = new SingleDelayedResult<>(); GrpcAsyncClientContext<KVServiceGrpc.KVServiceStub, Kv.Operand, Kv.Operand> context = rollKvCallModelFactory.createOperandToOperandContext(); context.setLatchInitCount(1) .setEndpoint(typeConversionUtils.toEndpoint(node)) .setFinishTimeout(RuntimeConstants.DEFAULT_WAIT_TIME, RuntimeConstants.DEFAULT_TIMEUNIT) .setCalleeStreamingMethodInvoker(KVServiceGrpc.KVServiceStub::delete) .setCallerStreamObserverClassAndArguments(StorageKvOperandToOperandObserver.class, delayedResult) .setGrpcMetadata(MetaConstants.createMetadataFromStoreInfo(storeInfo)); Kv.Operand result = null; GrpcStreamingClientTemplate<KVServiceGrpc.KVServiceStub, Kv.Operand, Kv.Operand> template = rollKvCallModelFactory.createOperandToOperandTemplate(); template.setGrpcAsyncClientContext(context); try { result = template.calleeStreamingRpcWithImmediateDelayedResult(request, delayedResult); } catch (InvocationTargetException e) { LOGGER.error(errorUtils.getStackTrace(e)); } return result; }
public void put(Kv.Operand operand, StoreInfo storeInfo, Node node) { GrpcAsyncClientContext<KVServiceGrpc.KVServiceStub, Kv.Operand, Kv.Empty> context = rollKvCallModelFactory.createOperandToEmptyContext(); context.setLatchInitCount(1) .setEndpoint(typeConversionUtils.toEndpoint(node)) .setFinishTimeout(RuntimeConstants.DEFAULT_WAIT_TIME, RuntimeConstants.DEFAULT_TIMEUNIT) .setCalleeStreamingMethodInvoker(KVServiceGrpc.KVServiceStub::put) .setCallerStreamObserverClassAndArguments(StorageKvPutResponseObserver.class) .setGrpcMetadata(MetaConstants.createMetadataFromStoreInfo(storeInfo)); GrpcStreamingClientTemplate<KVServiceGrpc.KVServiceStub, Kv.Operand, Kv.Empty> template = rollKvCallModelFactory.createOperandToEmptyTemplate(); template.setGrpcAsyncClientContext(context); template.calleeStreamingRpc(operand); }
public void destroy(Kv.Empty request, StoreInfo storeInfo, Node node) { GrpcAsyncClientContext<KVServiceGrpc.KVServiceStub, Kv.Empty, Kv.Empty> context = rollKvCallModelFactory.createEmptyToEmptyContext(); context.setLatchInitCount(1) .setEndpoint(typeConversionUtils.toEndpoint(node)) .setFinishTimeout(RuntimeConstants.DEFAULT_WAIT_TIME, RuntimeConstants.DEFAULT_TIMEUNIT) .setCalleeStreamingMethodInvoker(KVServiceGrpc.KVServiceStub::destroy) .setCallerStreamObserverClassAndArguments(StorageKvDestroyResponseObserver.class) .setGrpcMetadata(MetaConstants.createMetadataFromStoreInfo(storeInfo)); GrpcStreamingClientTemplate<KVServiceGrpc.KVServiceStub, Kv.Empty, Kv.Empty> template = rollKvCallModelFactory.createEmptyToEmptyTemplate(); template.setGrpcAsyncClientContext(context); template.calleeStreamingRpc(request); }
public OperandBroker iterate(Kv.Range request, StoreInfo storeInfo, Node node) { GrpcAsyncClientContext<KVServiceGrpc.KVServiceStub, Kv.Range, Kv.Operand> context = rollKvCallModelFactory.createRangeToOperandContext(); OperandBroker result = rollModelFactory.createOperandBroker(); context.setLatchInitCount(1) .setEndpoint(typeConversionUtils.toEndpoint(node)) .setFinishTimeout(RuntimeConstants.DEFAULT_WAIT_TIME, RuntimeConstants.DEFAULT_TIMEUNIT) .setCalleeStreamingMethodInvoker(KVServiceGrpc.KVServiceStub::iterate) .setCallerStreamObserverClassAndArguments(StorageKvIterateResponseStreamObserver.class, result) .setGrpcMetadata(MetaConstants.createMetadataFromStoreInfo(storeInfo)); GrpcStreamingClientTemplate<KVServiceGrpc.KVServiceStub, Kv.Range, Kv.Operand> template = rollKvCallModelFactory.createRangeToOperandTemplate(); template.setGrpcAsyncClientContext(context); template.calleeStreamingRpc(request); return result; }
public OperandBroker iterateStreaming(Kv.Range request, StoreInfo storeInfo, Node node) { GrpcAsyncClientContext<KVServiceGrpc.KVServiceStub, Kv.Range, Kv.Operand> context = rollKvCallModelFactory.createRangeToOperandContext(); OperandBroker result = rollModelFactory.createOperandBroker(); context.setLatchInitCount(1) .setEndpoint(typeConversionUtils.toEndpoint(node)) .setFinishTimeout(RuntimeConstants.DEFAULT_WAIT_TIME, RuntimeConstants.DEFAULT_TIMEUNIT) .setCalleeStreamingMethodInvoker(KVServiceGrpc.KVServiceStub::iterate) .setCallerStreamObserverClassAndArguments(StorageKvIterateResponseStreamObserver.class, result) .setGrpcMetadata(MetaConstants.createMetadataFromStoreInfo(storeInfo)); GrpcStreamingClientTemplate<KVServiceGrpc.KVServiceStub, Kv.Range, Kv.Operand> template = rollKvCallModelFactory.createRangeToOperandTemplate(); template.setGrpcAsyncClientContext(context); template.calleeStreamingRpcNoWait(request); return result; }
public void putAll(OperandBroker operandBroker, StoreInfo storeInfo, Node node) { GrpcAsyncClientContext<KVServiceGrpc.KVServiceStub, Kv.Operand, Kv.Empty> context = rollKvCallModelFactory.createOperandToEmptyContext(); context.setLatchInitCount(1) .setEndpoint(typeConversionUtils.toEndpoint(node)) .setFinishTimeout(RuntimeConstants.DEFAULT_WAIT_TIME, RuntimeConstants.DEFAULT_TIMEUNIT) .setCallerStreamingMethodInvoker(KVServiceGrpc.KVServiceStub::putAll) .setCallerStreamObserverClassAndArguments(StorageKvPutAllClientResponseStreamObserver.class) .setGrpcMetadata(MetaConstants.createMetadataFromStoreInfo(storeInfo)) .setRequestStreamProcessorClassAndArguments(StorageKvPutAllRequestStreamProcessor.class, operandBroker, node); GrpcStreamingClientTemplate<KVServiceGrpc.KVServiceStub, Kv.Operand, Kv.Empty> template = rollKvCallModelFactory.createOperandToEmptyTemplate(); template.setGrpcAsyncClientContext(context); template.initCallerStreamingRpc(); try { while (!operandBroker.isClosable()) { operandBroker.awaitLatch(RuntimeConstants.DEFAULT_WAIT_TIME, RuntimeConstants.DEFAULT_TIMEUNIT); template.processCallerStreamingRpc(); } } catch (Exception e) { template.errorCallerStreamingRpc(e); } template.completeStreamingRpc(); }