private UpdateRequestBuilder prepareUpdateFieldsOnDocument( String indexName, String documentId, Map<String, Object> fieldsToSet, Collection<String> fieldsToRemove, Map<String, String> fieldsToRename ) { if ((fieldsToSet == null || fieldsToSet.isEmpty()) && (fieldsToRemove == null || fieldsToRemove.isEmpty()) && (fieldsToRename == null || fieldsToRename.isEmpty())) { return null; } fieldsToSet = fieldsToSet == null ? Collections.emptyMap() : fieldsToSet.entrySet().stream() .collect(Collectors.toMap(e -> replaceFieldnameDots(e.getKey()), Map.Entry::getValue)); fieldsToRemove = fieldsToRemove == null ? Collections.emptyList() : fieldsToRemove.stream().map(this::replaceFieldnameDots).collect(Collectors.toList()); fieldsToRename = fieldsToRename == null ? Collections.emptyMap() : fieldsToRename.entrySet().stream() .collect(Collectors.toMap(e -> replaceFieldnameDots(e.getKey()), e -> replaceFieldnameDots(e.getValue()))); return getClient().prepareUpdate() .setIndex(indexName) .setId(documentId) .setType(getIdStrategy().getType()) .setScript(new Script( ScriptType.STORED, "painless", "updateFieldsOnDocumentScript", ImmutableMap.of("fieldsToSet", fieldsToSet, "fieldsToRemove", fieldsToRemove, "fieldsToRename", fieldsToRename) )) .setRetryOnConflict(MAX_RETRIES); }
private UpdateRequestBuilder prepareUpdateFieldsOnDocument( String indexName, String documentId, Map<String, Object> fieldsToSet, Collection<String> fieldsToRemove, Map<String, String> fieldsToRename ) { if ((fieldsToSet == null || fieldsToSet.isEmpty()) && (fieldsToRemove == null || fieldsToRemove.isEmpty()) && (fieldsToRename == null || fieldsToRename.isEmpty())) { return null; } fieldsToSet = fieldsToSet == null ? Collections.emptyMap() : fieldsToSet.entrySet().stream() .collect(Collectors.toMap(e -> replaceFieldnameDots(e.getKey()), Map.Entry::getValue)); fieldsToRemove = fieldsToRemove == null ? Collections.emptyList() : fieldsToRemove.stream().map(this::replaceFieldnameDots).collect(Collectors.toList()); fieldsToRename = fieldsToRename == null ? Collections.emptyMap() : fieldsToRename.entrySet().stream() .collect(Collectors.toMap(e -> replaceFieldnameDots(e.getKey()), e -> replaceFieldnameDots(e.getValue()))); return getClient().prepareUpdate() .setIndex(indexName) .setId(documentId) .setType(getIdStrategy().getType()) .setScript(new Script( ScriptType.STORED, "painless", "updateFieldsOnDocumentScript", ImmutableMap.of("fieldsToSet", fieldsToSet, "fieldsToRemove", fieldsToRemove, "fieldsToRename", fieldsToRename) )) .setRetryOnConflict(MAX_RETRIES); }
private UpdateRequestBuilder prepareRemoveFieldsFromDocument(String indexName, String documentId, Collection<String> fields) { List<String> fieldNames = fields.stream().map(this::replaceFieldnameDots).collect(Collectors.toList()); if (fieldNames.isEmpty()) { return null; } return getClient().prepareUpdate() .setIndex(indexName) .setId(documentId) .setType(getIdStrategy().getType()) .setScript(new Script( ScriptType.STORED, "painless", "deleteFieldsFromDocumentScript", ImmutableMap.of("fieldNames", fieldNames) )) .setRetryOnConflict(MAX_RETRIES); }
private UpdateRequestBuilder prepareRemoveFieldsFromDocument(String indexName, String documentId, Collection<String> fields) { List<String> fieldNames = fields.stream().map(this::replaceFieldnameDots).collect(Collectors.toList()); if (fieldNames.isEmpty()) { return null; } return getClient().prepareUpdate() .setIndex(indexName) .setId(documentId) .setType(getIdStrategy().getType()) .setScript(new Script( ScriptType.STORED, "painless", "deleteFieldsFromDocumentScript", ImmutableMap.of("fieldNames", fieldNames) )) .setRetryOnConflict(MAX_RETRIES); }
/** * Helper method to remove fields from source. This method will generate a ES update request. Retries on conflict. * * @param element Element that can be mapped to an ES document * @param fields fields to remove */ private void removeFieldsFromDocument(Element element, Collection<String> fields) { Map<String, Object> params = Maps.newHashMap(); int i = 0; StringBuilder script = new StringBuilder(); for (String field : fields) { String fieldName = "fieldName" + (i++); script.append("ctx._source.remove(").append(fieldName).append(");"); params.put(fieldName, field); } getClient().prepareUpdate() .setIndex(getIndexName(element)) .setId(element.getId()) .setType(ELEMENT_TYPE) .setScript( new Script( script.toString(), ScriptService.ScriptType.INLINE, null, params ) ) .setRetryOnConflict(MAX_RETRIES) .get(); }
/** * {@inheritDoc} */ @Override public void softDelete(final String type, final String id, final MetacatRequestContext metacatRequestContext) { try { RETRY_ES_PUBLISH.call(() -> { final XContentBuilder builder = XContentFactory.contentBuilder(contentType); builder.startObject().field(ElasticSearchDoc.Field.DELETED, true) .field(ElasticSearchDoc.Field.TIMESTAMP, java.time.Instant.now().toEpochMilli()) .field(ElasticSearchDoc.Field.USER, metacatRequestContext.getUserName()).endObject(); client.prepareUpdate(esIndex, type, id) .setRetryOnConflict(NO_OF_CONFLICT_RETRIES).setDoc(builder).get(esCallTimeout); ensureMigrationByCopy(type, Collections.singletonList(id)); return null; }); } catch (Exception e) { handleException("ElasticSearchUtil.softDelete", type, id, e, Metrics.CounterElasticSearchDelete.getMetricName()); } }
private void softDeleteDoc( final String type, final List<String> ids, final MetacatRequestContext metacatRequestContext) { try { RETRY_ES_PUBLISH.call(() -> { final BulkRequestBuilder bulkRequest = client.prepareBulk(); final XContentBuilder builder = XContentFactory.contentBuilder(contentType); builder.startObject().field(ElasticSearchDoc.Field.DELETED, true) .field(ElasticSearchDoc.Field.TIMESTAMP, java.time.Instant.now().toEpochMilli()) .field(ElasticSearchDoc.Field.USER, metacatRequestContext.getUserName()).endObject(); ids.forEach(id -> bulkRequest.add(client.prepareUpdate(esIndex, type, id) .setRetryOnConflict(NO_OF_CONFLICT_RETRIES).setDoc(builder))); final BulkResponse bulkResponse = bulkRequest.execute().actionGet(esBulkCallTimeout); if (bulkResponse.hasFailures()) { for (BulkItemResponse item : bulkResponse.getItems()) { if (item.isFailed()) { handleException("ElasticSearchUtil.bulkSoftDelete.item", type, item.getId(), item.getFailure().getCause(), Metrics.CounterElasticSearchDelete.getMetricName()); } } } return null; }); } catch (Exception e) { handleException("ElasticSearchUtil.bulkSoftDelete", type, ids, e, Metrics.CounterElasticSearchBulkDelete.getMetricName()); } }
.setType(ELEMENT_TYPE) .setScript(script, ScriptService.ScriptType.INLINE) .setRetryOnConflict(MAX_RETRIES) .setScriptParams(params);
ImmutableMap.of("fieldsToSet", Collections.emptyMap(), "fieldsToRemove", Collections.emptyList(), "fieldsToRename", fieldsToRename) )) .setRetryOnConflict(MAX_RETRIES) .request();
@Override public void markElementHidden(Graph graph, Element element, Visibility visibility, Authorizations authorizations) { try { String hiddenVisibilityPropertyName = addVisibilityToPropertyName(graph, HIDDEN_VERTEX_FIELD_NAME, visibility); if (!isPropertyInIndex(graph, hiddenVisibilityPropertyName)) { String indexName = getIndexName(element); IndexInfo indexInfo = ensureIndexCreatedAndInitialized(graph, indexName); addPropertyToIndex(graph, indexInfo, hiddenVisibilityPropertyName, visibility, Boolean.class, false); } XContentBuilder jsonBuilder = XContentFactory.jsonBuilder().startObject(); jsonBuilder.field(hiddenVisibilityPropertyName, true); jsonBuilder.endObject(); getClient() .prepareUpdate(getIndexName(element), ELEMENT_TYPE, element.getId()) .setDoc(jsonBuilder) .setRetryOnConflict(MAX_RETRIES) .get(); } catch (IOException e) { throw new VertexiumException("Could not mark element hidden", e); } }
@Override public void markPropertyHidden(Graph graph, Element element, Property property, Visibility visibility, Authorizations authorizations) { try { String hiddenVisibilityPropertyName = addVisibilityToPropertyName(graph, HIDDEN_PROPERTY_FIELD_NAME, visibility); String indexName = getIndexName(element); if (!isPropertyInIndex(graph, HIDDEN_PROPERTY_FIELD_NAME, visibility)) { IndexInfo indexInfo = ensureIndexCreatedAndInitialized(indexName); addPropertyToIndex(graph, indexInfo, hiddenVisibilityPropertyName, visibility, Boolean.class, false, false, false); } XContentBuilder jsonBuilder = XContentFactory.jsonBuilder().startObject(); jsonBuilder.field(hiddenVisibilityPropertyName, true); jsonBuilder.endObject(); getClient() .prepareUpdate(indexName, getIdStrategy().getType(), getIdStrategy().createElementDocId(element)) .setDoc(jsonBuilder) .setRetryOnConflict(MAX_RETRIES) .get(); getIndexRefreshTracker().pushChange(indexName); } catch (IOException e) { throw new VertexiumException("Could not mark element hidden", e); } }
private void updateDocs(final String type, final List<String> ids, final ObjectNode node) { try { RETRY_ES_PUBLISH.call(() -> { final BulkRequestBuilder bulkRequest = client.prepareBulk(); ids.forEach(id -> { bulkRequest.add(client.prepareUpdate(esIndex, type, id) .setRetryOnConflict(NO_OF_CONFLICT_RETRIES) .setDoc(metacatJson.toJsonAsBytes(node), XContentType.JSON)); }); final BulkResponse bulkResponse = bulkRequest.execute().actionGet(esBulkCallTimeout); if (bulkResponse.hasFailures()) { for (BulkItemResponse item : bulkResponse.getItems()) { if (item.isFailed()) { handleException("ElasticSearchUtil.updateDocs.item", type, item.getId(), item.getFailure().getCause(), Metrics.CounterElasticSearchUpdate.getMetricName()); } } } return null; }); } catch (Exception e) { handleException("ElasticSearchUtil.updatDocs", type, ids, e, Metrics.CounterElasticSearchBulkUpdate.getMetricName()); } }
@Override public void markPropertyHidden(Graph graph, Element element, Property property, Visibility visibility, Authorizations authorizations) { try { String hiddenVisibilityPropertyName = addVisibilityToPropertyName(graph, HIDDEN_PROPERTY_FIELD_NAME, visibility); String indexName = getIndexName(element); if (!isPropertyInIndex(graph, HIDDEN_PROPERTY_FIELD_NAME, visibility)) { IndexInfo indexInfo = ensureIndexCreatedAndInitialized(indexName); addPropertyToIndex(graph, indexInfo, hiddenVisibilityPropertyName, visibility, Boolean.class, false, false, false); } XContentBuilder jsonBuilder = XContentFactory.jsonBuilder().startObject(); jsonBuilder.field(hiddenVisibilityPropertyName, true); jsonBuilder.endObject(); getClient() .prepareUpdate(indexName, getIdStrategy().getType(), getIdStrategy().createElementDocId(element)) .setDoc(jsonBuilder) .setRetryOnConflict(MAX_RETRIES) .get(); getIndexRefreshTracker().pushChange(indexName); } catch (IOException e) { throw new VertexiumException("Could not mark element hidden", e); } }
@Override public void markElementHidden(Graph graph, Element element, Visibility visibility, Authorizations authorizations) { try { String hiddenVisibilityPropertyName = addVisibilityToPropertyName(graph, HIDDEN_VERTEX_FIELD_NAME, visibility); String indexName = getIndexName(element); if (!isPropertyInIndex(graph, HIDDEN_VERTEX_FIELD_NAME, visibility)) { IndexInfo indexInfo = ensureIndexCreatedAndInitialized(indexName); addPropertyToIndex(graph, indexInfo, hiddenVisibilityPropertyName, visibility, Boolean.class, false, false, false); } XContentBuilder jsonBuilder = XContentFactory.jsonBuilder().startObject(); jsonBuilder.field(hiddenVisibilityPropertyName, true); jsonBuilder.endObject(); getIndexRefreshTracker().pushChange(indexName); getClient() .prepareUpdate(indexName, getIdStrategy().getType(), getIdStrategy().createElementDocId(element)) .setDoc(jsonBuilder) .setRetryOnConflict(MAX_RETRIES) .get(); getIndexRefreshTracker().pushChange(indexName); } catch (IOException e) { throw new VertexiumException("Could not mark element hidden", e); } }
@Override public void markElementHidden(Graph graph, Element element, Visibility visibility, Authorizations authorizations) { try { String hiddenVisibilityPropertyName = addVisibilityToPropertyName(graph, HIDDEN_VERTEX_FIELD_NAME, visibility); String indexName = getIndexName(element); if (!isPropertyInIndex(graph, HIDDEN_VERTEX_FIELD_NAME, visibility)) { IndexInfo indexInfo = ensureIndexCreatedAndInitialized(indexName); addPropertyToIndex(graph, indexInfo, hiddenVisibilityPropertyName, visibility, Boolean.class, false, false, false); } XContentBuilder jsonBuilder = XContentFactory.jsonBuilder().startObject(); jsonBuilder.field(hiddenVisibilityPropertyName, true); jsonBuilder.endObject(); getIndexRefreshTracker().pushChange(indexName); getClient() .prepareUpdate(indexName, getIdStrategy().getType(), getIdStrategy().createElementDocId(element)) .setDoc(jsonBuilder) .setRetryOnConflict(MAX_RETRIES) .get(); getIndexRefreshTracker().pushChange(indexName); } catch (IOException e) { throw new VertexiumException("Could not mark element hidden", e); } }
private UpdateRequestBuilder buildUpdateRequestForReferencedFields(Entity parent, Entity child, boolean updateParent) { UpdateRequestBuilder urb = index.getClient() .prepareUpdate() .setIndex(index.getIndex(getLocalClass())) .setType(getLocalType()) .setRetryOnConflict(3) .setId(child.getId()); EntityDescriptor descriptor = index.getDescriptor(getLocalClass()); if (descriptor.hasRouting()) { Object routingKey = descriptor.getProperty(descriptor.getRouting()).writeToSource(child); if (Strings.isEmpty(routingKey)) { IndexAccess.LOG.WARN("Updating an entity of type %s (%s) without routing information!", child.getClass().getName(), child.getId()); } else { urb.setRouting(String.valueOf(routingKey)); } } Script script = computeUpdateScript(parent, updateParent); urb.setScript(script); if (IndexAccess.LOG.isFINE()) { IndexAccess.LOG.FINE("UPDATE: %s.%s: %s", index.getIndex(getLocalClass()), getLocalType(), script.toString()); } return urb; }
private UpdateRequestBuilder prepareUpdate(Graph graph, Element element, Authorizations authorizations) { try { IndexInfo indexInfo = addPropertiesToIndex(graph, element, element.getProperties()); XContentBuilder source = buildJsonContentFromElement(graph, element, authorizations); if (MUTATION_LOGGER.isTraceEnabled()) { MUTATION_LOGGER.trace("addElement json: %s: %s", element.getId(), source.string()); } getIndexRefreshTracker().pushChange(indexInfo.getIndexName()); return getClient() .prepareUpdate(indexInfo.getIndexName(), getIdStrategy().getType(), getIdStrategy().createElementDocId(element)) .setDocAsUpsert(true) .setDoc(source) .setRetryOnConflict(MAX_RETRIES); } catch (IOException e) { throw new VertexiumException("Could not add element", e); } }
private UpdateRequestBuilder prepareUpdate(Graph graph, Element element, Authorizations authorizations) { try { IndexInfo indexInfo = addPropertiesToIndex(graph, element, element.getProperties()); XContentBuilder source = buildJsonContentFromElement(graph, element, authorizations); if (MUTATION_LOGGER.isTraceEnabled()) { MUTATION_LOGGER.trace("addElement json: %s: %s", element.getId(), source.string()); } getIndexRefreshTracker().pushChange(indexInfo.getIndexName()); return getClient() .prepareUpdate(indexInfo.getIndexName(), getIdStrategy().getType(), getIdStrategy().createElementDocId(element)) .setDocAsUpsert(true) .setDoc(source) .setRetryOnConflict(MAX_RETRIES); } catch (IOException e) { throw new VertexiumException("Could not add element", e); } }
.setDocAsUpsert(true) .setDoc(source) .setRetryOnConflict(MAX_RETRIES); addActionRequestBuilderForFlush(element.getId(), updateRequestBuilder);
.setDocAsUpsert(true) .setDoc(source) .setRetryOnConflict(MAX_RETRIES); addActionRequestBuilderForFlush(element.getId(), updateRequestBuilder);