IndexerDefinitionBuilder builder = new IndexerDefinitionBuilder(); if (oldIndexerDef != null) builder.startFrom(oldIndexerDef); builder.name(indexerName); builder.lifecycleState(lifecycleState); builder.incrementalIndexingState(incrementalIdxState); builder.batchIndexingState(batchIdxState); builder.connectionType("solr"); builder.connectionParams(connectionParams); builder.indexerComponentFactory(indexerComponentFactoryOption.value(options)); builder.configuration(indexerConf); builder.defaultBatchIndexCliArguments(null); } else { builder.defaultBatchIndexCliArguments(defaultBatchIndexCliArguments); if (batchIndexCliArguments != null) { if (batchIndexCliArguments.length == 0) { builder.batchIndexCliArguments(null); } else { builder.batchIndexCliArguments(batchIndexCliArguments);
private void deleteIndexer(String indexerName) { // delete model boolean failedToDeleteIndexer = false; try { indexerModel.deleteIndexerInternal(indexerName); } catch (Throwable t) { log.error("Failed to delete indexer " + indexerName, t); failedToDeleteIndexer = true; } if (failedToDeleteIndexer) { try { IndexerDefinition indexer = indexerModel.getFreshIndexer(indexerName); indexer = new IndexerDefinitionBuilder().startFrom(indexer) .lifecycleState(IndexerDefinition.LifecycleState.DELETE_FAILED).build(); indexerModel.updateIndexerInternal(indexer); } catch (Throwable t) { log.error("Failed to set indexer state to " + IndexerDefinition.LifecycleState.DELETE_FAILED, t); } } }
IndexerDefinitionBuilder updatedIndexer = new IndexerDefinitionBuilder().startFrom(indexer); final String[] batchArguments = createBatchArguments(indexer); if (needsBatchBuildStart(indexer)) { .activeBatchBuildInfo(jobInfo) .batchIndexingState(BatchIndexingState.BUILDING) .batchIndexCliArguments(null) .build(); indexerModel.updateIndexerInternal(updatedIndexer.build());
private void unassignSubscription(String indexerName) { try { String lock = indexerModel.lockIndexer(indexerName); try { // Read current situation of record and assure it is still actual IndexerDefinition indexer = indexerModel.getFreshIndexer(indexerName); if (needsSubscriptionIdUnassigned(indexer)) { sepModel.removeSubscription(indexer.getSubscriptionId()); log.info("Deleted queue subscription for indexer " + indexerName); indexer = new IndexerDefinitionBuilder().startFrom(indexer).subscriptionId(null).build(); indexerModel.updateIndexerInternal(indexer); } } finally { indexerModel.unlockIndexer(lock); } } catch (Throwable t) { log.error("Error trying to delete the subscription for indexer " + indexerName, t); } }
indexerDefinitionBuilder.name(name); indexerDefinitionBuilder.lifecycleState(lifecycleState); indexerDefinitionBuilder.incrementalIndexingState(incrementalIndexingState); indexerDefinitionBuilder.batchIndexingState(batchIndexingState); indexerDefinitionBuilder.subscriptionId(queueSubscriptionId); indexerDefinitionBuilder.subscriptionTimestamp(subscriptionTimestamp); indexerDefinitionBuilder.configuration(configuration); indexerDefinitionBuilder.indexerComponentFactory(indexerComponentFactory); indexerDefinitionBuilder.connectionType(connectionType); indexerDefinitionBuilder.connectionParams(connectionParams); indexerDefinitionBuilder.activeBatchBuildInfo(activeBatchBuild); indexerDefinitionBuilder.lastBatchBuildInfo(lastBatchBuild); indexerDefinitionBuilder.batchIndexCliArguments(batchIndexCliArguments); indexerDefinitionBuilder.defaultBatchIndexCliArguments(defaultBatchIndexCliArguments); indexerDefinitionBuilder.occVersion(occVersion); return indexerDefinitionBuilder;
batchBuildInfo = batchBuildInfo.finishedSuccessfully(success); indexer = new IndexerDefinitionBuilder() .startFrom(indexer) .lastBatchBuildInfo(batchBuildInfo) .activeBatchBuildInfo(null) .batchIndexingState(IndexerDefinition.BatchIndexingState.INACTIVE) .build();
@Override public void jobStarted(String jobId, String trackingUrl) { try { // Lock internal bypasses the index-in-delete-state check, which does not matter (and might cause // failure) in our case. String lock = indexerModel.lockIndexerInternal(indexerName, false); try { IndexerDefinition definition = indexerModel.getFreshIndexer(indexerName); BatchBuildInfo batchBuildInfo = new BatchBuildInfo(definition.getActiveBatchBuildInfo()).withJob(jobId, trackingUrl); IndexerDefinition updatedDefinition = new IndexerDefinitionBuilder().startFrom(definition) .activeBatchBuildInfo(batchBuildInfo) .build(); indexerModel.updateIndexerInternal(updatedDefinition); log.info("Updated indexer batch build state for indexer " + indexerName); } finally { indexerModel.unlockIndexer(lock, true); } } catch (Exception e) { log.error("failed to update indexer batch build state for indexer " + indexerName); } } }
@Override public void addIndexer(IndexerDefinition indexer) throws IndexerExistsException, IndexerModelException, IndexerValidityException { assertValid(indexer); if (indexer.getIncrementalIndexingState() != IndexerDefinition.IncrementalIndexingState.DO_NOT_SUBSCRIBE) { indexer = new IndexerDefinitionBuilder().startFrom(indexer).subscriptionTimestamp(System.currentTimeMillis()).build(); } final String indexerPath = indexerCollectionPath + "/" + indexer.getName(); final byte[] data = IndexerDefinitionJsonSerDeser.INSTANCE.toJsonBytes(indexer); try { zk.retryOperation(new ZooKeeperOperation<String>() { @Override public String execute() throws KeeperException, InterruptedException { return zk.create(indexerPath, data, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); } }); } catch (KeeperException.NodeExistsException e) { throw new IndexerExistsException(indexer.getName()); } catch (Exception e) { throw new IndexerModelException("Error creating indexer.", e); } }
new IndexerDefinitionBuilder().startFrom(oldIndexer)).build();
private List<IndexerDefinition> getIndexersHttp(OptionSet options) throws Exception { HttpGet httpGet = new HttpGet(httpOption.value(options)); byte [] response = HttpUtil.sendRequest(httpGet); List<IndexerDefinitionBuilder> builders = IndexerDefinitionsJsonSerDeser.INSTANCE.fromJsonBytes(response); List<IndexerDefinition> indexers = new ArrayList<IndexerDefinition>(); for (IndexerDefinitionBuilder builder : builders) { indexers.add(builder.build()); } return indexers; }
private IndexerDefinition loadIndexer(String indexerName, boolean forCache) throws InterruptedException, KeeperException, IndexerNotFoundException { final String childPath = indexerCollectionPath + "/" + indexerName; final Stat stat = new Stat(); byte[] data; try { if (forCache) { // do not retry, install watcher data = zk.getData(childPath, watcher, stat); } else { // do retry, do not install watcher data = zk.retryOperation(new ZooKeeperOperation<byte[]>() { @Override public byte[] execute() throws KeeperException, InterruptedException { return zk.getData(childPath, false, stat); } }); } } catch (KeeperException.NoNodeException e) { throw new IndexerNotFoundException(indexerName); } IndexerDefinitionBuilder builder = IndexerDefinitionJsonSerDeser.INSTANCE.fromJsonBytes(data); builder.name(indexerName); builder.occVersion(stat.getVersion()); return builder.build(); }
public IndexerDefinitionBuilder fromJson(ObjectNode node) { return fromJson(node, new IndexerDefinitionBuilder()); }
indexerDefinitionBuilder.name(name); indexerDefinitionBuilder.lifecycleState(lifecycleState); indexerDefinitionBuilder.incrementalIndexingState(incrementalIndexingState); indexerDefinitionBuilder.batchIndexingState(batchIndexingState); indexerDefinitionBuilder.subscriptionId(queueSubscriptionId); indexerDefinitionBuilder.subscriptionTimestamp(subscriptionTimestamp); indexerDefinitionBuilder.configuration(configuration); indexerDefinitionBuilder.indexerComponentFactory(indexerComponentFactory); indexerDefinitionBuilder.connectionType(connectionType); indexerDefinitionBuilder.connectionParams(connectionParams); indexerDefinitionBuilder.activeBatchBuildInfo(activeBatchBuild); indexerDefinitionBuilder.lastBatchBuildInfo(lastBatchBuild); indexerDefinitionBuilder.batchIndexCliArguments(batchIndexCliArguments); indexerDefinitionBuilder.defaultBatchIndexCliArguments(defaultBatchIndexCliArguments); indexerDefinitionBuilder.occVersion(occVersion); return indexerDefinitionBuilder;
batchBuildInfo = batchBuildInfo.finishedSuccessfully(success); indexer = new IndexerDefinitionBuilder() .startFrom(indexer) .lastBatchBuildInfo(batchBuildInfo) .activeBatchBuildInfo(null) .batchIndexingState(BatchIndexingState.INACTIVE) .build();
private void unassignSubscription(String indexerName) { try { String lock = indexerModel.lockIndexer(indexerName); try { // Read current situation of record and assure it is still actual IndexerDefinition indexer = indexerModel.getFreshIndexer(indexerName); if (needsSubscriptionIdUnassigned(indexer)) { sepModel.removeSubscription(indexer.getSubscriptionId()); log.info("Deleted queue subscription for indexer " + indexerName); indexer = new IndexerDefinitionBuilder().startFrom(indexer).subscriptionId(null).build(); indexerModel.updateIndexerInternal(indexer); } } finally { indexerModel.unlockIndexer(lock); } } catch (Throwable t) { log.error("Error trying to delete the subscription for indexer " + indexerName, t); } }
@Override public void jobStarted(String jobId, String trackingUrl) { try { // Lock internal bypasses the index-in-delete-state check, which does not matter (and might cause // failure) in our case. String lock = indexerModel.lockIndexerInternal(indexerName, false); try { IndexerDefinition definition = indexerModel.getFreshIndexer(indexerName); BatchBuildInfo batchBuildInfo = new BatchBuildInfo(definition.getActiveBatchBuildInfo()).withJob(jobId, trackingUrl); IndexerDefinition updatedDefinition = new IndexerDefinitionBuilder().startFrom(definition) .activeBatchBuildInfo(batchBuildInfo) .build(); indexerModel.updateIndexerInternal(updatedDefinition); log.info("Updated indexer batch build state for indexer " + indexerName); } finally { indexerModel.unlockIndexer(lock, true); } } catch (Exception e) { log.error("failed to update indexer batch build state for indexer " + indexerName); } } }
@Override public void addIndexer(IndexerDefinition indexer) throws IndexerExistsException, IndexerModelException, IndexerValidityException { assertValid(indexer); if (indexer.getIncrementalIndexingState() != IndexerDefinition.IncrementalIndexingState.DO_NOT_SUBSCRIBE) { indexer = new IndexerDefinitionBuilder().startFrom(indexer).subscriptionTimestamp(System.currentTimeMillis()).build(); } final String indexerPath = indexerCollectionPath + "/" + indexer.getName(); final byte[] data = IndexerDefinitionJsonSerDeser.INSTANCE.toJsonBytes(indexer); try { zk.retryOperation(new ZooKeeperOperation<String>() { @Override public String execute() throws KeeperException, InterruptedException { return zk.create(indexerPath, data, CreateMode.PERSISTENT); } }); } catch (KeeperException.NodeExistsException e) { throw new IndexerExistsException(indexer.getName()); } catch (Exception e) { throw new IndexerModelException("Error creating indexer.", e); } }
new IndexerDefinitionBuilder().startFrom(oldIndexer)).build();
private IndexerDefinition getFreshIndexer(String indexerName, String path) throws Exception { // FixMe: should we have an individual GetIndexer command so we don't have to iterate? HttpGet httpGet = new HttpGet(path); byte [] response = HttpUtil.sendRequest(httpGet); List<IndexerDefinitionBuilder> builders = IndexerDefinitionsJsonSerDeser.INSTANCE.fromJsonBytes(response); for (IndexerDefinitionBuilder builder : builders) { IndexerDefinition def = builder.build(); if (indexerName.equals(def.getName())) { return def; } } return null; } }
private IndexerDefinition loadIndexer(String indexerName, boolean forCache) throws InterruptedException, KeeperException, IndexerNotFoundException { final String childPath = indexerCollectionPath + "/" + indexerName; final Stat stat = new Stat(); byte[] data; try { if (forCache) { // do not retry, install watcher data = zk.getData(childPath, watcher, stat); } else { // do retry, do not install watcher data = zk.retryOperation(new ZooKeeperOperation<byte[]>() { @Override public byte[] execute() throws KeeperException, InterruptedException { return zk.getData(childPath, false, stat); } }); } } catch (KeeperException.NoNodeException e) { throw new IndexerNotFoundException(indexerName); } IndexerDefinitionBuilder builder = IndexerDefinitionJsonSerDeser.INSTANCE.fromJsonBytes(data); builder.name(indexerName); builder.occVersion(stat.getVersion()); return builder.build(); }