@Override protected void mapNativeValue( final GeoWaveInputKey key, final Object value, final org.apache.hadoop.mapreduce.Mapper<GeoWaveInputKey, ObjectWritable, GeoWaveInputKey, ObjectWritable>.Context context) throws IOException, InterruptedException { @SuppressWarnings("unchecked") final AnalyticItemWrapper<T> wrapper = itemWrapperFactory.create((T) value); outputKey.setInternalAdapterId(key.getInternalAdapterId()); outputKey.setDataId( new ByteArray( StringUtils.stringToBinary(nestedGroupCentroidAssigner.getGroupForLevel(wrapper)))); outputKey.setGeoWaveKey(key.getGeoWaveKey()); context.write(outputKey, currentValue); }
@Override public void partitionWith(final PartitionData partitionData) throws Exception { outputValue.setInternalAdapterId(key.getInternalAdapterId()); AdapterWithObjectWritable.fillWritableWithAdapter( serializationTool, outputValue, key.getInternalAdapterId(), key.getDataId(), unwrappedValue); partitionDataWritable.setPartitionData(partitionData); context.write(partitionDataWritable, outputValue); } });
/** * Helper method to create an object writable from a value managed by the adapter. * * @param key * @param value * @return */ protected ObjectWritable toWritableValue(final GeoWaveInputKey key, final Object value) { return serializationTool.toWritable(key.getInternalAdapterId(), value); }
final MapContext<GeoWaveInputKey, GridCoverage, GeoWaveInputKey, Object> context) throws IOException, InterruptedException { if (helper.isOriginalCoverage(key.getInternalAdapterId())) { final InternalDataAdapter<?> adapter = super.serializationTool.getInternalAdapter(key.getInternalAdapterId()); if ((adapter != null) && (adapter.getAdapter() != null) new GeoWaveKeyImpl( helper.getNewDataId(c), key.getInternalAdapterId(), partitionKey, sortKey, 0); final GeoWaveInputKey inputKey = new GeoWaveInputKey(helper.getNewAdapterId(), geowaveKey, helper.getIndexName()); context.write(inputKey, c);
protected Pair<GeoWaveInputKey, T> valueToEntry(final GeoWaveRow row, final Object value) { final short adapterId = row.getAdapterId(); final T result = (T) (isOutputWritable ? serializationTool.getHadoopWritableSerializerForAdapter(adapterId).toWritable(value) : value); final GeoWaveInputKey key = new GeoWaveInputKey(row, index.getName()); return Pair.of(key, result); }
private static float getProgressForRange( final GeoWaveRowRange range, final GeoWaveInputKey currentKey) { if (currentKey == null) { return 0f; } if ((range != null) && (range.getStartSortKey() != null) && (range.getEndSortKey() != null) && (currentKey.getGeoWaveKey() != null)) { // TODO GEOWAVE-1018 this doesn't account for partition keys at all // just look at the row progress return getProgressForRange( range.getStartSortKey(), range.getEndSortKey(), GeoWaveKey.getCompositeId(currentKey.getGeoWaveKey())); } // if we can't figure it out, then claim no progress return 0f; }
@Override public int getPartition( final GeoWaveInputKey key, final ObjectWritable val, final int numPartitions) { final byte[] grpIDInBytes = KeyManager.getGroup(key.getDataId().getBytes()); final int partition = hash(grpIDInBytes) % numPartitions; return partition; }
final Pair<byte[], byte[]> pair = key.getPartitionAndSortKey(index); mergedCoverage = newAdapter.getCoverageFromRasterTile(
@Override protected void map( final GeoWaveInputKey key, final SimpleFeature value, final Mapper<GeoWaveInputKey, SimpleFeature, AvroKey<AvroSimpleFeatureCollection>, NullWritable>.Context context) throws IOException, InterruptedException { AvroSFCWriter avroWriter = adapterIdToAvroWriterMap.get(key.getInternalAdapterId()); if (avroWriter == null) { avroWriter = new AvroSFCWriter(value.getFeatureType(), batchSize); adapterIdToAvroWriterMap.put(key.getInternalAdapterId(), avroWriter); } final AvroSimpleFeatureCollection retVal = avroWriter.write(value); if (retVal != null) { outKey.datum(retVal); context.write(outKey, outVal); } }
@Override protected void reduceNativeValues( final GeoWaveInputKey key, final Iterable<Object> values, final Reducer<GeoWaveInputKey, ObjectWritable, GeoWaveOutputKey, T>.Context context) throws IOException, InterruptedException { final String groupID = KeyManager.getGroupAsString(key.getDataId().getBytes()); for (final Object value : values) { final AnalyticItemWrapper<T> sampleItem = itemWrapperFactory.create((T) value); Integer outputCount = outputCounts.get(groupID); outputCount = outputCount == null ? Integer.valueOf(0) : outputCount; if ((outputCount == null) || (outputCount < maxCount)) { final AnalyticItemWrapper<T> centroid = createCentroid(groupID, sampleItem); if (centroid != null) { context.write( new GeoWaveOutputKey(sampleDataTypeName, indexNames), centroid.getWrappedItem()); outputCount++; outputCounts.put(groupID, outputCount); } } } }
@Override protected void mapNativeValue( final GeoWaveInputKey key, final Object value, final org.apache.hadoop.mapreduce.Mapper<GeoWaveInputKey, ObjectWritable, GeoWaveInputKey, ObjectWritable>.Context context) throws IOException, InterruptedException { @SuppressWarnings("unchecked") final double rank = samplingFunction.rank(sampleSize, (T) value); if (rank > 0.0000000001) { final AnalyticItemWrapper<Object> wrapper = itemWrapperFactory.create(value); outputKey.setDataId( new ByteArray( keyManager.putData( nestedGroupCentroidAssigner.getGroupForLevel(wrapper), 1.0 - rank, // sorts // in // ascending // order key.getDataId().getBytes()))); outputKey.setInternalAdapterId(key.getInternalAdapterId()); outputKey.setGeoWaveKey(key.getGeoWaveKey()); context.write(outputKey, currentValue); } }
@Override protected void reduceNativeValues( final GeoWaveInputKey key, final Iterable<Object> values, final Reducer<GeoWaveInputKey, ObjectWritable, GeoWaveOutputKey, Object>.Context context) throws IOException, InterruptedException { outputKey.setTypeName(internalAdapterStore.getTypeName(key.getInternalAdapterId())); for (final Object value : values) { context.write(outputKey, value); } }
final String groupID = StringUtils.stringFromBinary(key.getDataId().getBytes()); final AnalyticItemWrapper<T> centroid = centroidManager.getCentroid(groupID); for (final Object value : values) {
protected void mapWritableValue( final GeoWaveInputKey key, final ObjectWritable value, final Mapper<GeoWaveInputKey, ObjectWritable, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException { mapNativeValue(key, serializationTool.fromWritable(key.getInternalAdapterId(), value), context); }
private SimpleFeature getSimpleFeature(final GeoWaveInputKey key, final Object entry) { final Geometry geometry = dimExtractor.getGeometry(entry); final double[] extraDims = dimExtractor.getDimensions(entry); final String inputID = StringUtils.stringFromBinary(key.getDataId().getBytes()); final SimpleFeature pointFeature = AnalyticFeature.createGeometryFeature( outputAdapter.getFeatureType(), batchID, inputID, inputID, groupID, 0.0, geometry, dimExtractor.getDimensionNames(), extraDims, 1, 1, 0); return pointFeature; }
@Override public void write(final GeoWaveInputKey key, final Object value) throws IOException, InterruptedException { context.write(key, serializationTool.toWritable(key.getInternalAdapterId(), value)); }
@Override public void write(final GeoWaveInputKey key, final Object value) throws IOException, InterruptedException { writableContext.write(key, serializationTool.toWritable(key.getInternalAdapterId(), value)); }
protected void reduceWritableValues( final GeoWaveInputKey key, final Iterable<ObjectWritable> values, final Reducer<GeoWaveInputKey, ObjectWritable, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException { final HadoopWritableSerializer<?, Writable> serializer = serializationTool.getHadoopWritableSerializerForAdapter(key.getInternalAdapterId()); final Iterable<Object> transformedValues = Iterables.transform(values, new Function<ObjectWritable, Object>() { @Override public Object apply(final ObjectWritable writable) { final Object innerObj = writable.get(); return (innerObj instanceof Writable) ? serializer.fromWritable((Writable) innerObj) : innerObj; } }); reduceNativeValues(key, transformedValues, context); }
final T unwrappedValue = (T) ((value instanceof ObjectWritable) ? serializationTool.fromWritable(key.getInternalAdapterId(), (ObjectWritable) value) : value); try {