@Override public void sink(FlowProcess<Properties> flowProcess, SinkCall<Object[], Object> sinkCall) throws IOException { client.writeToIndex(sinkCall); } }
RestRepository repository = new RestRepository(settings); if (repository.touch()) { if (repository.waitForYellow()) { log.warn(String.format("Timed out waiting for index [%s] to reach yellow health", resource)); String clientNode = repository.getRestClient().getCurrentNode(); if (log.isDebugEnabled()) { log.debug(String.format("Client-node routing detected; partition writer instance [%s] assigned to [%s]", Map<ShardInfo, NodeInfo> targetShards = repository.getWriteTargetPrimaryShards(settings.getNodesClientOnly()); repository.close(); repository = new RestRepository(settings);
private void flushNoAck() { writer.repository.flush(); }
public static void checkIndexExistence(Settings settings) { // Only open a connection and check if autocreate is disabled if (!settings.getIndexAutoCreate()) { RestRepository repository = new RestRepository(settings); try { doCheckIndexExistence(settings, repository); } finally { repository.close(); } } }
@Override public void close() { if (isOpen()) { // TODO: look at collecting these stats some other way later. if (clientSettings.getBatchRefreshAfterWrite()) { writeClient.getRestClient().refresh(endpoint); } writeClient.close(); } } }
InitializationUtils.filterNonIngestNodesIfNeeded(settings, log); RestRepository client = new RestRepository(settings); try { boolean indexExists = client.indexExists(true); shards = client.getReadTargetShards(); if (log.isTraceEnabled()) { log.trace("Creating splits for shards " + shards); mapping = client.getMappings(); if (log.isDebugEnabled()) { log.debug(String.format("Discovered resolved mapping {%s} for [%s]", mapping.getResolvedView(), settings.getResourceRead())); partitions = findSlicePartitions(client.getRestClient(), settings, mapping, nodesMap, shards, log); } else { partitions = findShardPartitions(settings, mapping, nodesMap, shards, log); return partitions; } finally { client.close();
public static <T> void saveSchemaIfNeeded(Object conf, ValueWriter<T> schemaWriter, T schema, Log log) { Settings settings = HadoopSettingsManager.loadFrom(conf); if (settings.getIndexAutoCreate()) { RestRepository client = new RestRepository(settings); if (!client.indexExists(false)) { if (schemaWriter == null) { log.warn(String.format("No mapping found [%s] and no schema found; letting Elasticsearch perform auto-mapping...", settings.getResourceWrite())); } else { log.info(String.format("No mapping found [%s], creating one based on given schema", settings.getResourceWrite())); ContentBuilder builder = ContentBuilder.generate(schemaWriter).value(schema).flush(); BytesArray content = ((FastByteArrayOutputStream) builder.content()).bytes(); builder.close(); client.putMapping(content); if (log.isDebugEnabled()) { log.debug(String.format("Creating ES mapping [%s] from schema [%s]", content.toString(), schema)); } } } client.close(); } }
InitializationUtils.filterNonIngestNodesIfNeeded(settings, log); RestRepository client = new RestRepository(settings); try { boolean indexExists = client.resourceExists(true); shards = client.getReadTargetShards(); if (log.isTraceEnabled()) { log.trace("Creating splits for shards " + shards); mapping = client.getMappings(); if (log.isDebugEnabled()) { log.debug(String.format("Discovered resolved mapping {%s} for [%s]", mapping.getResolvedView(), settings.getResourceRead())); partitions = findSlicePartitions(client.getRestClient(), settings, mapping, nodesMap, shards, log); } else { partitions = findShardPartitions(settings, mapping, nodesMap, shards, log); return partitions; } finally { client.close();
ScrollQuery sq = scanAll(scanQuery, null, scrollReader); try { BytesArray entry = new BytesArray(0); entry.add(StringUtils.toUTF(String.format(baseFormat, encodedId))); writeProcessedToIndex(entry); flush();
@Override public void close() { if (!closed) { closed = true; repository.close(); } } }
@Override public void close() { if (!closed) { closed = true; finished = true; batch = Collections.emptyList(); reader.close(); // typically the scroll is closed after it is consumed so this will trigger a 404 // however we're closing it either way if (StringUtils.hasText(scrollId)) { repository.getRestClient().deleteScroll(scrollId); } } }
/** * Creates a RestRepository for use with a multi-index resource pattern. The client is left pinned * to the original node that it was pinned to since the shard locations cannot be determined at all. * @param settings Job settings * @param currentInstance Partition number * @param resource Configured write resource * @param log Logger to use * @return The RestRepository to be used by the partition writer */ private static RestRepository initMultiIndices(Settings settings, long currentInstance, Resource resource, Log log) { if (log.isDebugEnabled()) { log.debug(String.format("Resource [%s] resolves as an index pattern", resource)); } // multi-index write - since we don't know before hand what index will be used, use an already selected node String node = SettingsUtils.getPinnedNode(settings); if (log.isDebugEnabled()) { log.debug(String.format("Partition writer instance [%s] assigned to [%s]", currentInstance, node)); } return new RestRepository(settings); }
ValueReader reader = ObjectUtils.instantiate(settings.getSerializerValueReaderClassName(), settings); RestRepository repository = new RestRepository(settings); Mapping fieldMapping = null; if (StringUtils.hasText(partition.getSerializedMapping())) { String clientNode = repository.getRestClient().getCurrentNode(); if (log.isDebugEnabled()) { log.debug(String.format("Client-node routing detected; partition reader instance [%s] assigned to [%s]", if (QueryUtils.isExplicitlyRequested(partition.getIndex(), indices) == false) { IndicesAliases indicesAliases = new GetAliasesRequestBuilder(repository.getRestClient()) .indices(partition.getIndex()) .execute().getIndices();
private RestRepository mockRepository() throws Exception { Map<String, Object> data = new HashMap<String, Object>(); data.put("field", "value"); String id = "1"; Object[] hit = new Object[]{id, data}; RestRepository mocked = Mockito.mock(RestRepository.class); ScrollReader.Scroll start = new ScrollReader.Scroll("abcd", 10, Collections.<Object[]>emptyList(), 5, 5); Mockito.doReturn(start).when(mocked).scroll(Matchers.anyString(), Matchers.any(BytesArray.class), Matchers.any(ScrollReader.class)); ScrollReader.Scroll middle = new ScrollReader.Scroll("efgh", 10, Collections.<Object[]>emptyList(), 3, 3); Mockito.doReturn(middle).when(mocked).scroll(Matchers.eq("abcd"), Matchers.any(ScrollReader.class)); ScrollReader.Scroll end = new ScrollReader.Scroll("ijkl", 10, Collections.singletonList(hit), 2, 1); Mockito.doReturn(end).when(mocked).scroll(Matchers.eq("efgh"), Matchers.any(ScrollReader.class)); ScrollReader.Scroll finalScroll = new ScrollReader.Scroll("mnop", 10, true); Mockito.doReturn(finalScroll).when(mocked).scroll(Matchers.eq("ijkl"), Matchers.any(ScrollReader.class)); RestClient mockClient = Mockito.mock(RestClient.class); Mockito.when(mockClient.deleteScroll(Matchers.eq("mnop"))).thenReturn(true); Mockito.when(mockClient.deleteScroll(Matchers.anyString())).thenReturn(false); Mockito.doReturn(mockClient).when(mocked).getRestClient(); return mocked; } }
public static void checkIndexExistence(Settings settings, RestRepository client) { // check index existence if (!settings.getIndexAutoCreate()) { if (client == null) { client = new RestRepository(settings); } if (!client.indexExists(false)) { client.close(); throw new EsHadoopIllegalArgumentException(String.format("Target index [%s] does not exist and auto-creation is disabled [setting '%s' is '%s']", settings.getResourceWrite(), ConfigurationOptions.ES_INDEX_AUTO_CREATE, settings.getIndexAutoCreate())); } } }
RestRepository client = new RestRepository(settings); MappingSet mappings = client.getMappings(); Mapping mapping = mappings.isEmpty() ? null : mappings.getResolvedView(); Collection<String> fields = CascadingUtils.fieldToAlias(settings, getSourceFields());
public boolean isEmpty(boolean read) { Resource res = (read ? resources.getResourceRead() : resources.getResourceWrite()); boolean exists = client.indexExists(res.index()); return (exists ? count(read) <= 0 : true); }
public List<List<Map<String, Object>>> getReadTargetShards() { for (int retries = 0; retries < 3; retries++) { List<List<Map<String, Object>>> result = doGetReadTargetShards(); if (result != null) { return result; } } throw new EsHadoopIllegalStateException("Cluster state volatile; cannot find node backing shards - please check whether your cluster is stable"); }