private List<IndexerDefinition> getIndexersHttp(OptionSet options) throws Exception { HttpGet httpGet = new HttpGet(httpOption.value(options)); byte [] response = HttpUtil.sendRequest(httpGet); List<IndexerDefinitionBuilder> builders = IndexerDefinitionsJsonSerDeser.INSTANCE.fromJsonBytes(response); List<IndexerDefinition> indexers = new ArrayList<IndexerDefinition>(); for (IndexerDefinitionBuilder builder : builders) { indexers.add(builder.build()); } return indexers; }
private IndexerDefinition getFreshIndexer(String indexerName, String path) throws Exception { // FixMe: should we have an individual GetIndexer command so we don't have to iterate? HttpGet httpGet = new HttpGet(path); byte [] response = HttpUtil.sendRequest(httpGet); List<IndexerDefinitionBuilder> builders = IndexerDefinitionsJsonSerDeser.INSTANCE.fromJsonBytes(response); for (IndexerDefinitionBuilder builder : builders) { IndexerDefinition def = builder.build(); if (indexerName.equals(def.getName())) { return def; } } return null; } }
@Override public void run(OptionSet options) throws Exception { super.run(options); IndexerDefinition indexer = null; try { IndexerDefinitionBuilder builder = buildIndexerDefinition(options, null); indexer = builder.build(); } catch (IllegalArgumentException e) { System.err.printf("Error adding indexer: %s\n", e.getMessage()); return; } model.addIndexer(indexer); System.out.println("Indexer added: " + indexer.getName()); } }
private IndexerDefinition getIndexerFromJson(byte [] jsonBytes) throws IndexerServerException { try { return IndexerDefinitionJsonSerDeser.INSTANCE.fromJsonBytes(jsonBytes).build(); } catch (Exception e) { throw new IndexerServerException(HttpServletResponse.SC_UNAUTHORIZED, new SentryBindingException("Unable to test permissions, "+ e.getMessage(), e)); } }
public UpdateIndexerInput fromJsonBytes(byte [] jsonBytes) { ObjectNode node = null; try { node = (ObjectNode) new ObjectMapper().readTree(new ByteArrayInputStream(jsonBytes)); } catch (IOException e) { throw new JsonFormatException("Error parsing indexer definition JSON.", e); } if (node.size() != 2) { new JsonFormatException("Expected object field of size 2, got size " + node.size()); } IndexerDefinition baseIndexer = getDefinitionFromNode(node,"baseIndexer").build(); IndexerDefinition newIndexer = getDefinitionFromNode(node, "newIndexer").build(); return new UpdateIndexerInput(newIndexer, baseIndexer); }
private void updateIndexerHttp(OptionSet options, String indexerName) throws Exception { String path = httpOption.value(options); IndexerDefinition indexer = getFreshIndexer(indexerName, path); if (indexer == null) { throw new CliException("Indexer does not exist: " + indexerName); } IndexerDefinitionBuilder builder = buildIndexerDefinition(options, indexer); IndexerDefinition newIndexer = builder.build(); UpdateIndexerInput input = new UpdateIndexerInput(newIndexer, indexer); byte [] jsonBytes = UpdateIndexerInputJsonSerDeser.INSTANCE.toJsonBytes(input); String putPath = path + (path.endsWith("/")?"":"/") + indexerName; HttpPut httpPut = new HttpPut(putPath); httpPut.setEntity(new ByteArrayEntity(jsonBytes)); byte [] response = HttpUtil.sendRequest(httpPut); }
@Override public void run(OptionSet options) throws Exception { super.run(options); IndexerDefinition indexer = null; try { IndexerDefinitionBuilder builder = buildIndexerDefinition(options, null); indexer = builder.build(); } catch (IllegalArgumentException e) { System.err.printf("Error adding indexer: %s\n", e.getMessage()); return; } if (!options.has("http")) { model.addIndexer(indexer); System.out.println("Indexer added: " + indexer.getName()); } else { addIndexerHttp(options, indexer); } }
private void deleteIndexer(String indexerName) { // delete model boolean failedToDeleteIndexer = false; try { indexerModel.deleteIndexerInternal(indexerName); } catch (Throwable t) { log.error("Failed to delete indexer " + indexerName, t); failedToDeleteIndexer = true; } if (failedToDeleteIndexer) { try { IndexerDefinition indexer = indexerModel.getFreshIndexer(indexerName); indexer = new IndexerDefinitionBuilder().startFrom(indexer) .lifecycleState(IndexerDefinition.LifecycleState.DELETE_FAILED).build(); indexerModel.updateIndexerInternal(indexer); } catch (Throwable t) { log.error("Failed to set indexer state to " + IndexerDefinition.LifecycleState.DELETE_FAILED, t); } } }
/** * Add an indexer definition */ @POST @Consumes("application/json") @Produces("application/json") public byte[] post(@Context SecurityContext securityContext, byte [] jsonBytes) throws IndexerServerException { WriteableIndexerModel model = getModel(); IndexerDefinitionBuilder builder = getBuilderFromJson(jsonBytes, null); IndexerDefinition indexer = builder.build(); try { model.addIndexer(indexer); } catch (IndexerExistsException iee) { throw new IndexerServerException(HttpServletResponse.SC_BAD_REQUEST, iee); } catch (IndexerValidityException ive) { throw new IndexerServerException(HttpServletResponse.SC_BAD_REQUEST, ive); } catch (IndexerModelException ime) { throw new IndexerServerException(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, ime); } return toJsonResultBytes("Indexer added: " + indexer.getName()); }
private void deleteIndexer(String indexerName) { // delete model boolean failedToDeleteIndexer = false; try { indexerModel.deleteIndexerInternal(indexerName); } catch (Throwable t) { log.error("Failed to delete indexer " + indexerName, t); failedToDeleteIndexer = true; } if (failedToDeleteIndexer) { try { IndexerDefinition indexer = indexerModel.getFreshIndexer(indexerName); indexer = new IndexerDefinitionBuilder().startFrom(indexer) .lifecycleState(IndexerDefinition.LifecycleState.DELETE_FAILED).build(); indexerModel.updateIndexerInternal(indexer); } catch (Throwable t) { log.error("Failed to set indexer state to " + IndexerDefinition.LifecycleState.DELETE_FAILED, t); } } }
private void unassignSubscription(String indexerName) { try { String lock = indexerModel.lockIndexer(indexerName); try { // Read current situation of record and assure it is still actual IndexerDefinition indexer = indexerModel.getFreshIndexer(indexerName); if (needsSubscriptionIdUnassigned(indexer)) { sepModel.removeSubscription(indexer.getSubscriptionId()); log.info("Deleted queue subscription for indexer " + indexerName); indexer = new IndexerDefinitionBuilder().startFrom(indexer).subscriptionId(null).build(); indexerModel.updateIndexerInternal(indexer); } } finally { indexerModel.unlockIndexer(lock); } } catch (Throwable t) { log.error("Error trying to delete the subscription for indexer " + indexerName, t); } }
private IndexerDefinition loadIndexer(String indexerName, boolean forCache) throws InterruptedException, KeeperException, IndexerNotFoundException { final String childPath = indexerCollectionPath + "/" + indexerName; final Stat stat = new Stat(); byte[] data; try { if (forCache) { // do not retry, install watcher data = zk.getData(childPath, watcher, stat); } else { // do retry, do not install watcher data = zk.retryOperation(new ZooKeeperOperation<byte[]>() { @Override public byte[] execute() throws KeeperException, InterruptedException { return zk.getData(childPath, false, stat); } }); } } catch (KeeperException.NoNodeException e) { throw new IndexerNotFoundException(indexerName); } IndexerDefinitionBuilder builder = IndexerDefinitionJsonSerDeser.INSTANCE.fromJsonBytes(data); builder.name(indexerName); builder.occVersion(stat.getVersion()); return builder.build(); }
@Override public void jobStarted(String jobId, String trackingUrl) { try { // Lock internal bypasses the index-in-delete-state check, which does not matter (and might cause // failure) in our case. String lock = indexerModel.lockIndexerInternal(indexerName, false); try { IndexerDefinition definition = indexerModel.getFreshIndexer(indexerName); BatchBuildInfo batchBuildInfo = new BatchBuildInfo(definition.getActiveBatchBuildInfo()).withJob(jobId, trackingUrl); IndexerDefinition updatedDefinition = new IndexerDefinitionBuilder().startFrom(definition) .activeBatchBuildInfo(batchBuildInfo) .build(); indexerModel.updateIndexerInternal(updatedDefinition); log.info("Updated indexer batch build state for indexer " + indexerName); } finally { indexerModel.unlockIndexer(lock, true); } } catch (Exception e) { log.error("failed to update indexer batch build state for indexer " + indexerName); } } }
private void unassignSubscription(String indexerName) { try { String lock = indexerModel.lockIndexer(indexerName); try { // Read current situation of record and assure it is still actual IndexerDefinition indexer = indexerModel.getFreshIndexer(indexerName); if (needsSubscriptionIdUnassigned(indexer)) { sepModel.removeSubscription(indexer.getSubscriptionId()); log.info("Deleted queue subscription for indexer " + indexerName); indexer = new IndexerDefinitionBuilder().startFrom(indexer).subscriptionId(null).build(); indexerModel.updateIndexerInternal(indexer); } } finally { indexerModel.unlockIndexer(lock); } } catch (Throwable t) { log.error("Error trying to delete the subscription for indexer " + indexerName, t); } }
@Override public void jobStarted(String jobId, String trackingUrl) { try { // Lock internal bypasses the index-in-delete-state check, which does not matter (and might cause // failure) in our case. String lock = indexerModel.lockIndexerInternal(indexerName, false); try { IndexerDefinition definition = indexerModel.getFreshIndexer(indexerName); BatchBuildInfo batchBuildInfo = new BatchBuildInfo(definition.getActiveBatchBuildInfo()).withJob(jobId, trackingUrl); IndexerDefinition updatedDefinition = new IndexerDefinitionBuilder().startFrom(definition) .activeBatchBuildInfo(batchBuildInfo) .build(); indexerModel.updateIndexerInternal(updatedDefinition); log.info("Updated indexer batch build state for indexer " + indexerName); } finally { indexerModel.unlockIndexer(lock, true); } } catch (Exception e) { log.error("failed to update indexer batch build state for indexer " + indexerName); } } }
newIndexer = builder.build();
private void assignSubscription(String indexerName) { try { String lock = indexerModel.lockIndexer(indexerName); try { // Read current situation of record and assure it is still actual IndexerDefinition indexer = indexerModel.getFreshIndexer(indexerName); if (needsSubscriptionIdAssigned(indexer)) { // We assume we are the only process which creates subscriptions which begin with the // prefix "Indexer:". This way we are sure there are no naming conflicts or conflicts // due to concurrent operations (e.g. someone deleting this subscription right after we // created it). String subscriptionId = subscriptionId(indexer.getName()); sepModel.addSubscription(subscriptionId); indexer = new IndexerDefinitionBuilder().startFrom(indexer).subscriptionId(subscriptionId).build(); indexerModel.updateIndexerInternal(indexer); log.info("Assigned subscription ID '" + subscriptionId + "' to indexer '" + indexerName + "'"); } } finally { indexerModel.unlockIndexer(lock); } } catch (Throwable t) { log.error("Error trying to assign a subscription to index " + indexerName, t); } }
@Override public void addIndexer(IndexerDefinition indexer) throws IndexerExistsException, IndexerModelException, IndexerValidityException { assertValid(indexer); if (indexer.getIncrementalIndexingState() != IndexerDefinition.IncrementalIndexingState.DO_NOT_SUBSCRIBE) { indexer = new IndexerDefinitionBuilder().startFrom(indexer).subscriptionTimestamp(System.currentTimeMillis()).build(); } final String indexerPath = indexerCollectionPath + "/" + indexer.getName(); final byte[] data = IndexerDefinitionJsonSerDeser.INSTANCE.toJsonBytes(indexer); try { zk.retryOperation(new ZooKeeperOperation<String>() { @Override public String execute() throws KeeperException, InterruptedException { return zk.create(indexerPath, data, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); } }); } catch (KeeperException.NodeExistsException e) { throw new IndexerExistsException(indexer.getName()); } catch (Exception e) { throw new IndexerModelException("Error creating indexer.", e); } }
@Override public void addIndexer(IndexerDefinition indexer) throws IndexerExistsException, IndexerModelException, IndexerValidityException { assertValid(indexer); if (indexer.getIncrementalIndexingState() != IndexerDefinition.IncrementalIndexingState.DO_NOT_SUBSCRIBE) { indexer = new IndexerDefinitionBuilder().startFrom(indexer).subscriptionTimestamp(System.currentTimeMillis()).build(); } final String indexerPath = indexerCollectionPath + "/" + indexer.getName(); final byte[] data = IndexerDefinitionJsonSerDeser.INSTANCE.toJsonBytes(indexer); try { zk.retryOperation(new ZooKeeperOperation<String>() { @Override public String execute() throws KeeperException, InterruptedException { return zk.create(indexerPath, data, CreateMode.PERSISTENT); } }); } catch (KeeperException.NodeExistsException e) { throw new IndexerExistsException(indexer.getName()); } catch (Exception e) { throw new IndexerModelException("Error creating indexer.", e); } }
@Override protected void run(OptionSet options) throws Exception { super.run(options); String indexerName = nameOption.value(options); if (!model.hasIndexer(indexerName)) { throw new CliException("Indexer does not exist: " + indexerName); } IndexerDefinition indexerDef = model.getIndexer(indexerName); if (indexerDef.getLifecycleState() == LifecycleState.DELETE_REQUESTED || indexerDef.getLifecycleState() == LifecycleState.DELETING) { System.err.printf("Delete of '%s' is already in progress\n", indexerName); return; } IndexerDefinitionBuilder builder = new IndexerDefinitionBuilder(); builder.startFrom(indexerDef); builder.lifecycleState(LifecycleState.DELETE_REQUESTED); model.updateIndexerInternal(builder.build()); waitForDeletion(indexerName); }