private void testFlowSpec(FlowConfig flowConfig) { try { FlowConfigResourceLocalHandler.createFlowSpecForConfig(flowConfig); } catch (FlowConfigLoggedException e) { Assert.fail("Should not get to here"); } }
/** * Add flowConfig locally and trigger all listeners */ public CreateResponse createFlowConfig(FlowConfig flowConfig) throws FlowConfigLoggedException { return this.createFlowConfig(flowConfig, true); }
/** * Method to handle add flow config message forwarded by Helix (Standby) node. * In load balance mode, the FlowCatalog I/O was handled on standby when receiving Restli, so only need to handle * {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onDeleteSpec(URI, String, Properties)} part. * Otherwise, we have to handle both FlowCatalog I/O and {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onDeleteSpec(URI, String, Properties)}. * * Please refer to {@link FlowConfigResourceLocalHandler#deleteFlowConfig(FlowId, Properties)}. It will handle both FlowCatalog I/O and * {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onDeleteSpec(URI, String, Properties)} in non-balance mode. */ private void handleDelete(String msg) throws IOException { try { FlowId id = FlowConfigUtils.deserializeFlowId(msg); if (flowCatalogLocalCommit) { // in balance mode, flow spec is already deleted in flow catalog on standby node. URI flowUri = FlowConfigResourceLocalHandler.createFlowSpecUri(id); log.info("Only handle update {} scheduling because flow catalog is committed locally on standby.", flowUri); jobScheduler.onDeleteSpec(flowUri, FlowSpec.Builder.DEFAULT_VERSION); } else { resourceHandler.deleteFlowConfig(id, new Properties()); } } catch (URISyntaxException e) { throw new IOException(e); } }
/** * Update flowConfig locally and trigger all listeners */ public UpdateResponse updateFlowConfig(FlowId flowId, FlowConfig flowConfig) throws FlowConfigLoggedException { return updateFlowConfig(flowId, flowConfig, true); }
@Override public void configure(Binder binder) { binder.bind(FlowConfigsResourceHandler.class) .annotatedWith(Names.named(FlowConfigsResource.INJECT_FLOW_CONFIG_RESOURCE_HANDLER)) .toInstance(new FlowConfigResourceLocalHandler(flowCatalog)); // indicate that we are in unit testing since the resource is being blocked until flow catalog changes have // been made binder.bindConstant().annotatedWith(Names.named(FlowConfigsResource.INJECT_READY_TO_USE)).to(Boolean.TRUE); binder.bind(RequesterService.class) .annotatedWith(Names.named(FlowConfigsResource.INJECT_REQUESTER_SERVICE)).toInstance(new NoopRequesterService(config)); } });
/** * Delete flowConfig locally and trigger all listeners */ public UpdateResponse deleteFlowConfig(FlowId flowId, Properties header) throws FlowConfigLoggedException { return deleteFlowConfig(flowId, header, true); }
@Override public FlowConfig getFlowConfig(FlowId flowId) throws FlowConfigLoggedException { return this.localHandler.getFlowConfig(flowId); }
this.localHandler.updateFlowConfig(flowId, flowConfig, false); return new UpdateResponse(HttpStatus.S_200_OK); } else { return this.localHandler.updateFlowConfig(flowId, flowConfig);
new FlowConfigResourceLocalHandler(this.flowCatalog), this.helixManager, this.scheduler);
this.localHandler.deleteFlowConfig(flowId, header, false); return this.localHandler.deleteFlowConfig(flowId, header);
@Override public FlowConfig getFlowConfig(FlowId flowId) throws FlowConfigLoggedException { return this.localHandler.getFlowConfig(flowId); }
/** * Method to handle add flow config message forwarded by Helix (Standby) node. * In load balance mode, the FlowCatalog I/O was handled on standby when receiving Restli, so only need to handle * {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onAddSpec(Spec)} part. * Otherwise, we have to handle both FlowCatalog I/O and {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onAddSpec(Spec)}. * * Please refer to {@link FlowConfigResourceLocalHandler#createFlowConfig(FlowConfig)}. It will handle both FlowCatalog I/O and * {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onAddSpec(Spec)} in non-balance mode. */ private void handleAdd(String msg) throws IOException { FlowConfig config = FlowConfigUtils.deserializeFlowConfig(msg); if (this.flowCatalogLocalCommit) { // in balance mode, flow spec is already added in flow catalog on standby node. FlowSpec flowSpec = FlowConfigResourceLocalHandler.createFlowSpecForConfig(config); log.info("Only handle add {} scheduling because flow catalog is committed locally on standby.", flowSpec); jobScheduler.onAddSpec(flowSpec); } else { resourceHandler.createFlowConfig(config); } }
this.localHandler.updateFlowConfig(flowId, flowConfig, false); return new UpdateResponse(HttpStatus.S_200_OK); } else { return this.localHandler.updateFlowConfig(flowId, flowConfig);
new FlowConfigResourceLocalHandler(this.flowCatalog), this.helixManager, this.scheduler);
if (this.flowCatalogLocalCommit) { this.localHandler.createFlowConfig(flowConfig, false); return this.localHandler.createFlowConfig(flowConfig);
this.localHandler.deleteFlowConfig(flowId, header, false); return this.localHandler.deleteFlowConfig(flowId, header);
/** * Delete flowConfig locally and trigger all listeners iff @param triggerListener is set to true */ public UpdateResponse deleteFlowConfig(FlowId flowId, Properties header, boolean triggerListener) throws FlowConfigLoggedException { log.info("[GAAS-REST] Delete called with flowGroup {} flowName {}", flowId.getFlowGroup(), flowId.getFlowName()); URI flowUri = null; try { flowUri = createFlowSpecUri(flowId); this.flowCatalog.remove(flowUri, header, triggerListener); return new UpdateResponse(HttpStatus.S_200_OK); } catch (URISyntaxException e) { throw new FlowConfigLoggedException(HttpStatus.S_400_BAD_REQUEST, "bad URI " + flowUri, e); } }
/** * Method to handle add flow config message forwarded by Helix (Standby) node. * In load balance mode, the FlowCatalog I/O was handled on standby when receiving Restli, so only need to handle * {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onUpdateSpec(Spec)} part. * Otherwise, we have to handle both FlowCatalog I/O and {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onUpdateSpec(Spec)}. * * Please refer to {@link FlowConfigResourceLocalHandler#updateFlowConfig(FlowId, FlowConfig)}. It will handle both FlowCatalog I/O and * {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onUpdateSpec(Spec)} in non-balance mode. */ private void handleUpdate(String msg) throws IOException { FlowConfig config = FlowConfigUtils.deserializeFlowConfig(msg); if (flowCatalogLocalCommit) { // in balance mode, flow spec is already updated in flow catalog on standby node. FlowSpec flowSpec = FlowConfigResourceLocalHandler.createFlowSpecForConfig(config); log.info("Only handle update {} scheduling because flow catalog is committed locally on standby.", flowSpec); jobScheduler.onUpdateSpec(flowSpec); } else { resourceHandler.updateFlowConfig(config.getId(), config); } }
if (this.flowCatalogLocalCommit) { this.localHandler.createFlowConfig(flowConfig, false); return this.localHandler.createFlowConfig(flowConfig);
/** * Method to handle add flow config message forwarded by Helix (Standby) node. * In load balance mode, the FlowCatalog I/O was handled on standby when receiving Restli, so only need to handle * {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onDeleteSpec(URI, String, Properties)} part. * Otherwise, we have to handle both FlowCatalog I/O and {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onDeleteSpec(URI, String, Properties)}. * * Please refer to {@link FlowConfigResourceLocalHandler#deleteFlowConfig(FlowId, Properties)}. It will handle both FlowCatalog I/O and * {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onDeleteSpec(URI, String, Properties)} in non-balance mode. */ private void handleDelete(String msg) throws IOException { try { FlowId id = FlowConfigUtils.deserializeFlowId(msg); if (flowCatalogLocalCommit) { // in balance mode, flow spec is already deleted in flow catalog on standby node. URI flowUri = FlowConfigResourceLocalHandler.createFlowSpecUri(id); log.info("Only handle update {} scheduling because flow catalog is committed locally on standby.", flowUri); jobScheduler.onDeleteSpec(flowUri, FlowSpec.Builder.DEFAULT_VERSION); } else { resourceHandler.deleteFlowConfig(id, new Properties()); } } catch (URISyntaxException e) { throw new IOException(e); } }