@Override public void loadOneRow() throws IOException { Trip t = new Trip(); t.sourceFileLine = row + 1; // offset line number by 1 to account for 0-based row index t.route_id = getStringField("route_id", true); t.service_id = getStringField("service_id", true); t.trip_id = getStringField("trip_id", true); t.trip_headsign = getStringField("trip_headsign", false); t.trip_short_name = getStringField("trip_short_name", false); t.direction_id = getIntField("direction_id", false, 0, 1); t.block_id = getStringField("block_id", false); // make a blocks multimap t.shape_id = getStringField("shape_id", false); t.bikes_allowed = getIntField("bikes_allowed", false, 0, 2); t.wheelchair_accessible = getIntField("wheelchair_accessible", false, 0, 2); t.feed = feed; t.feed_id = feed.feedId; feed.trips.put(t.trip_id, t); /* Check referential integrity without storing references. Trip cannot directly reference Services or Routes because they would be serialized into the MapDB. */ // TODO confirm existence of shape ID getRefField("service_id", true, feed.services); getRefField("route_id", true, feed.routes); }
@Override public void loadOneRow() throws IOException { StopTime st = new StopTime(); st.sourceFileLine = row + 1; // offset line number by 1 to account for 0-based row index st.trip_id = getStringField("trip_id", true); // TODO: arrival_time and departure time are not required, but if one is present the other should be // also, if this is the first or last stop, they are both required st.arrival_time = getTimeField("arrival_time", false); st.departure_time = getTimeField("departure_time", false); st.stop_id = getStringField("stop_id", true); st.stop_sequence = getIntField("stop_sequence", true, 0, Integer.MAX_VALUE); st.stop_headsign = getStringField("stop_headsign", false); st.pickup_type = getIntField("pickup_type", false, 0, 3); // TODO add ranges as parameters st.drop_off_type = getIntField("drop_off_type", false, 0, 3); st.shape_dist_traveled = getDoubleField("shape_dist_traveled", false, 0D, Double.MAX_VALUE); // FIXME using both 0 and NaN for "missing", define DOUBLE_MISSING st.timepoint = getIntField("timepoint", false, 0, 1, INT_MISSING); st.feed = null; // this could circular-serialize the whole feed feed.stop_times.put(new Fun.Tuple2(st.trip_id, st.stop_sequence), st); /* Check referential integrity without storing references. StopTime cannot directly reference Trips or Stops because they would be serialized into the MapDB. */ getRefField("trip_id", true, feed.trips); getRefField("stop_id", true, feed.stops); }
@Override public byte[] put(Long key, byte[] value) { byte[] b = this.treeMap.put(key, value); this.db.commit(); return b; }
@Override public void put(byte[] key, byte[] val) { btree.put(key, val); }
@Override public void add( String nodeKey, String propertyName, T value ) { logger.trace("Adding node '{0}' to '{1}' index with value '{2}'", nodeKey, name, value); keysByValue.put(value, nodeKey); }
@Override public void add( String nodeKey, String propertyName, T value ) { logger.trace("Adding node '{0}' to '{1}' index with value '{2}'", nodeKey, name, value); keysByValue.put(value, nodeKey); }
/** * {@inheritDoc} */ @Override protected void putRaw(String id, Map<Object, Object> object) { this.db.treeMap(COLLECTION_NAME).put(id, object); }
@Override public void modify( final @NotNull Object[] key, final @Nullable byte[] oldValue, final @Nullable byte[] newValue, final boolean triggered ) { if (newValue == null) { final Object[] oldKey = keyFunction.apply(oldValue); index.remove(oldKey); } else { final Object[] newKey = keyFunction.apply(newValue); index.put(newKey, newValue); } } }
public static void main(String[] args) { //import org.mapdb.* DB db = DBMaker.memoryDB().make(); BTreeMap<byte[], Integer> map = db .treeMap("towns", Serializer.BYTE_ARRAY, Serializer.INTEGER) .createOrOpen(); map.put("New York".getBytes(), 1); map.put("New Jersey".getBytes(), 2); map.put("Boston".getBytes(), 3); //get all New* cities Map<byte[], Integer> newCities = map.prefixSubMap("New".getBytes()); } }
public void save(SpatialDataItem obj) { map.put(obj.id, obj); for(Tuple3<Integer, Integer, Long> tuple : obj.getTiles(Z_INDEX)) { tileIndex.add(tuple); } db.commit(); }
@Override public Map<String, Object> updateItem( final String tableName, final MapStoreKey key, final Map<String, Object> payload ) { final BTreeMap<Object[], byte[]> table = getTable(tableName, key); final Object[] keyArray = convertSimpleKeyToArray(key); // As a precondition, the item is guaranteed to be present final Map<String, Object> returnValue = new TreeMap<>(getItem(tableName, key).get()); payload.entrySet().stream().forEach(entry -> { if (isNullOrEmptyEntry().test(entry)) { returnValue.remove(entry.getKey()); } else { returnValue.put(entry.getKey(), entry.getValue()); } }); try { table.put(keyArray, OBJECT_MAPPER.writeValueAsBytes(returnValue)); } catch (IOException e) { throw new RuntimeException(e); } return returnValue; }
public void save(List<SpatialDataItem> objs) { for(SpatialDataItem obj : objs) { if (map.containsKey(obj.id)) continue; map.put(obj.id, obj); for (Tuple3<Integer, Integer, Long> tuple : obj.getTiles(Z_INDEX)) { tileIndex.add(tuple); } } db.commit(); }
@Override public Map<String, Object> putItem( final String tableName, final MapStoreKey key, final Map<String, Object> payload ) { final BTreeMap<Object[], byte[]> table = getTable(tableName, key); final Object[] keyArray = convertSimpleKeyToArray(key); final Map<String, Object> returnValue = new TreeMap<>(payload); returnValue.putIfAbsent(key.getHashField(), key.getHashValue()); key.getRangeField().ifPresent(rangeField -> returnValue.putIfAbsent(rangeField, key.getRangeKey().getValue())); try { table.put(keyArray, OBJECT_MAPPER.writeValueAsBytes(returnValue)); } catch (IOException e) { throw new RuntimeException(e); } return returnValue; }
public void save(SpeedSample speedSample) { synchronized (this) { int week = SegmentStatistics.getWeekSinceEpoch(speedSample.getTime()); int hour = SegmentStatistics.getHourOfWeek(speedSample.getTime()); weekSet.add(week); Fun.Tuple2<Long, Integer> sampleId = new Fun.Tuple2<>(speedSample.getSegmentId(), week); SegmentStatistics segmentStatistics; segmentStatistics = statsMap.get(sampleId); if(segmentStatistics == null) segmentStatistics = new SegmentStatistics(); segmentStatistics.addSample(speedSample); statsMap.put(sampleId, segmentStatistics); } }
@Override public synchronized void addRecords( JournalRecord... records ) { if (stopped) { return; } LOGGER.debug("Adding {0} records", records.length); for (JournalRecord record : records) { if (record.getTimeBasedKey() < 0) { //generate a unique timestamp only if there isn't one. In some scenarios (i.e. running in a cluster) we //always want to keep the original TS because otherwise it would be impossible to have a correct order //and therefore search long createTimeMillisUTC = TIME_BASED_KEYS.nextKey(); record.withTimeBasedKey(createTimeMillisUTC); } this.records.put(record.getTimeBasedKey(), record); } this.journalDB.commit(); }
@Override public synchronized void addRecords( JournalRecord... records ) { if (stopped) { return; } LOGGER.debug("Adding {0} records", records.length); for (JournalRecord record : records) { if (record.getTimeBasedKey() < 0) { //generate a unique timestamp only if there isn't one. In some scenarios (i.e. running in a cluster) we //always want to keep the original TS because otherwise it would be impossible to have a correct order //and therefore search long createTimeMillisUTC = TIME_BASED_KEYS.nextKey(); record.withTimeBasedKey(createTimeMillisUTC); } this.records.put(record.getTimeBasedKey(), record); } this.journalDB.commit(); }
void write() { final int rndByteMax = RND_MB.length - valSize; int rndByteOffset = 0; for (final int key : keys) { if (intKey) { wkb.putInt(0, key, LITTLE_ENDIAN); } else { wkb.putStringWithoutLengthUtf8(0, padKey(key)); } if (valRandom) { wvb.putBytes(0, RND_MB, rndByteOffset, valSize); rndByteOffset += valSize; if (rndByteOffset >= rndByteMax) { rndByteOffset = 0; } } else { wvb.putInt(0, key); } map.put(wkb.byteArray(), wvb.byteArray()); } } }
@Override public boolean createIndex( final MapStoreIndexName indexName, final MapStoreTableKey indexKey ) { // Create a new index table using the map store final MapDBIndexMetadata indexMetadata = indexMetadata(indexName, indexKey); final boolean createdIndex = mapStore.createTable(indexMetadata.toString(), indexKey); if (createdIndex) { // Copy all table records to the new index final Function<byte[], Object[]> indexKeyFunction = indexKeyFunction(indexMetadata); final BTreeMap<?, byte[]> table = mapDB.treeMap(indexName.getTableName()) .valueSerializer(Serializer.BYTE_ARRAY) .open(); final BTreeMap<Object[], byte[]> index = mapDB.treeMap(indexMetadata.toString()) .keySerializer(createKeySerializer(indexKey.getHashFieldType(), indexKey.getRangeFieldType())) .valueSerializer(Serializer.BYTE_ARRAY) .open(); table.getValues().forEach((byte[] value) -> { final Object[] newKey = indexKeyFunction.apply(value); index.put(newKey, value); }); } return createdIndex; }
@Override public void loadOneRow() throws IOException { StopTime st = new StopTime(); st.id = row + 1; // offset line number by 1 to account for 0-based row index st.trip_id = getStringField("trip_id", true); // TODO: arrival_time and departure time are not required, but if one is present the other should be // also, if this is the first or last stop, they are both required st.arrival_time = getTimeField("arrival_time", false); st.departure_time = getTimeField("departure_time", false); st.stop_id = getStringField("stop_id", true); st.stop_sequence = getIntField("stop_sequence", true, 0, Integer.MAX_VALUE); st.stop_headsign = getStringField("stop_headsign", false); st.pickup_type = getIntField("pickup_type", false, 0, 3); // TODO add ranges as parameters st.drop_off_type = getIntField("drop_off_type", false, 0, 3); st.shape_dist_traveled = getDoubleField("shape_dist_traveled", false, 0D, Double.MAX_VALUE); // FIXME using both 0 and NaN for "missing", define DOUBLE_MISSING st.timepoint = getIntField("timepoint", false, 0, 1, INT_MISSING); st.feed = null; // this could circular-serialize the whole feed feed.stop_times.put(new Fun.Tuple2(st.trip_id, st.stop_sequence), st); /* Check referential integrity without storing references. StopTime cannot directly reference Trips or Stops because they would be serialized into the MapDB. */ getRefField("trip_id", true, feed.trips); getRefField("stop_id", true, feed.stops); }