private Resource makeProtoResource(final ProtoSerializable field, final String name) { // We automatically get the correct adapter for whatever type 'field' happens to be final ProtoAdapter adapter = field.getProtoAdapter(); // The adapter handles all the actual serialization using the protobuf classes. Easy! final byte[] byteContents = adapter.serialize(field); final ByteArrayResource resource = new ByteArrayResource(byteContents.length) .withName(name); try (BufferedOutputStream out = new BufferedOutputStream(resource.write())) { out.write(byteContents); } catch (final Exception e) { throw new CoreException("Could not convert {} to a readable resource.", field, e); } return resource; }
@Test public void testSerialization() throws IOException, ClassNotFoundException { final MultiMap<Polygon, Polygon> map = getMultiMap(); final WritableResource out = new ByteArrayResource(); final ObjectOutputStream outStream = new ObjectOutputStream(out.write()); outStream.writeObject(map); Streams.close(outStream); try (ObjectInputStream inStream = new ObjectInputStream(out.read())) { @SuppressWarnings("unchecked") final MultiMap<Polygon, Polygon> result = (MultiMap<Polygon, Polygon>) inStream .readObject(); Assert.assertEquals(map, result); } catch (final Exception e) { throw e; } } }
@Test public void testReadWriteConsistency() { final WritableResource resource = new ByteArrayResource(); final ProtoAtlasBuilder protoAtlasBuilder = new ProtoAtlasBuilder(); final PackedAtlasBuilder packedAtlasBuilder = setUpTestAtlasBuilder(); // make sure the atlases are the same final Atlas outAtlas = packedAtlasBuilder.get(); protoAtlasBuilder.write(outAtlas, resource); final Atlas inAtlas = protoAtlasBuilder.read(resource); logger.debug("ATLAS THAT WAS WRITTEN OUT:\n{}\n", outAtlas); logger.debug("ATLAS THAT WAS READ BACK:\n{}\n", inAtlas); Assert.assertEquals(outAtlas, inAtlas); }
@Test public void testSerialization() throws ClassNotFoundException { final MultiPolygon map = new MultiPolygon(MultiMapTest.getMultiMap()); final WritableResource out = new ByteArrayResource(); try (ObjectOutputStream outStream = new ObjectOutputStream(out.write())) { outStream.writeObject(map); } catch (final IOException e) { throw new CoreException("Unable to write to {}", out, e); } try (ObjectInputStream inStream = new ObjectInputStream(out.read())) { final MultiPolygon result = (MultiPolygon) inStream.readObject(); Assert.assertEquals(map, result); } catch (final IOException e) { throw new CoreException("Unable to read from {}", out, e); } }
@Test(expected = CoreException.class) public void testInvalidFileFormat() { final byte[] values = { 1, 2, 3 }; final ByteArrayResource resource = new ByteArrayResource(); resource.writeAndClose(values); final Atlas atlas = new AtlasResourceLoader().load(resource); atlas.metaData(); }
final ByteArrayResource originalFileBytes = new ByteArrayResource(); originalFileBytes.copyFrom(new InputStreamResource( () -> CachingTests.class.getResourceAsStream(FEATURE_JSON))); final ByteArrayResource fileBytesCacheMiss = new ByteArrayResource(); fileBytesCacheMiss.copyFrom(cache.get(resourceUri).get()); final byte[] fileBytesCacheMissArray = fileBytesCacheMiss.readBytesAndClose(); final ByteArrayResource fileBytesCacheHit = new ByteArrayResource(); fileBytesCacheHit.copyFrom(cache.get(resourceUri).get()); final byte[] fileBytesCacheHitArray = fileBytesCacheHit.readBytesAndClose();
@Test public void testSize() { final PackedAtlas atlas = new PackedAtlasTest().getAtlas(); final ByteArrayResource zipped = new ByteArrayResource(8192).withName("zippedByteArray"); zipped.setCompressor(Compressor.GZIP); atlas.save(zipped); logger.info("Zipped Size: {}", zipped.length()); }
private Atlas deserialized() { final Atlas atlas = new PackedAtlasTest().getAtlas(); final ByteArrayResource resource = new ByteArrayResource(524288) .withName("testSerializationByteArray"); atlas.save(resource); return PackedAtlas.load(resource); }
@Test public void testSaveWhenBuildingFromScratch() { final StringList countries = new StringList(); countries.add("DMA"); final CountryBoundaryMap boundaries = CountryBoundaryMap .fromPlainText(new InputStreamResource(() -> CountryToShardListCacheTest.class .getResourceAsStream("DMA_boundary.txt"))); final Sharding sharding = Sharding.forString("dynamic@" + CountryToShardListCacheTest.class .getResource("tree-6-14-100000.txt.gz").getPath()); // test building the cache from scratch final CountryToShardListCache cache = new CountryToShardListCache(boundaries, countries, sharding); final ByteArrayResource output = new ByteArrayResource(); cache.save(output); Assert.assertEquals("DMA||9-168-233, 9-169-233, 9-168-234, 10-338-468", output.all()); } }
@Test public void testSaveWhenBuildingFromFile() { final CountryToShardListCache cache = new CountryToShardListCache(this.countryToShardList); // test that you get the same file back that was used to initialize the cache final ByteArrayResource output = new ByteArrayResource(); cache.save(output); Assert.assertEquals(this.countryToShardList.all(), output.all()); }
public void testSizes() throws Exception { final long size = 100000; final ByteArrayResource javaOutput = new ByteArrayResource().withName("Java"); final ByteArrayResource zippedOutput = new ByteArrayResource().withName("Zipped"); javaOutput.setCompressor(Compressor.GZIP); zippedOutput.setCompressor(Compressor.GZIP); final TwoStringObject object = new TwoStringObject(size); final Resource oneResource = new StringResource(object.getOne()).withName("one"); final Resource twoResource = new StringResource(object.getTwo()).withName("two"); final ObjectOutputStream out = new ObjectOutputStream(javaOutput.write()); out.writeObject(object); Streams.close(out); new ZipWritableResource(zippedOutput).writeAndClose(oneResource, twoResource); System.out.println(javaOutput.getName() + " " + javaOutput.length()); System.out.println(zippedOutput.getName() + " " + zippedOutput.length()); }
@Test public void testEmptyTags() { final HashMap<String, String> map = new HashMap<>(); final PackedAtlasBuilder builder = new PackedAtlasBuilder(); builder.addPoint(1, new Location(Latitude.ZERO, Longitude.ZERO), map); final ByteArrayResource protoResource = new ByteArrayResource(LARGE_DEFAULT_SIZE); final PackedAtlas atlas = (PackedAtlas) builder.get(); logger.info("Starting serialization process..."); Time protoTime = Time.now(); atlas.setSaveSerializationFormat(AtlasSerializationFormat.PROTOBUF); atlas.save(protoResource); logger.info("Proto serialization time: {}", protoTime.elapsedSince()); logger.info("Proto resource size: {} bytes ({} KiB)", protoResource.length(), protoResource.length() / BYTES_PER_KIBIBYTE); protoTime = Time.now(); final PackedAtlas loadedFromProto = PackedAtlas.load(protoResource); logger.info("Proto deserialization time: {}", protoTime.elapsedSince()); Assert.assertEquals(atlas, loadedFromProto); }
@Test public void testConsistency() { final PackedAtlas atlas = getNewAtlas(); final ByteArrayResource javaResource = new ByteArrayResource(LARGE_DEFAULT_SIZE); final ByteArrayResource protoResource = new ByteArrayResource(LARGE_DEFAULT_SIZE); logger.info("Starting serialization process..."); Time javaTime = Time.now(); atlas.setSaveSerializationFormat(AtlasSerializationFormat.JAVA); atlas.save(javaResource); logger.info("Java serialization time: {}", javaTime.elapsedSince()); Time protoTime = Time.now(); atlas.setSaveSerializationFormat(AtlasSerializationFormat.PROTOBUF); atlas.save(protoResource); logger.info("Proto serialization time: {}", protoTime.elapsedSince()); logger.info("Java resource size: {} bytes ({} KiB)", javaResource.length(), javaResource.length() / BYTES_PER_KIBIBYTE); logger.info("Proto resource size: {} bytes ({} KiB)", protoResource.length(), protoResource.length() / BYTES_PER_KIBIBYTE); javaTime = Time.now(); final PackedAtlas loadedFromJava = PackedAtlas.load(javaResource); logger.info("Java deserialization time: {}", javaTime.elapsedSince()); protoTime = Time.now(); final PackedAtlas loadedFromProto = PackedAtlas.load(protoResource); logger.info("Proto deserialization time: {}", protoTime.elapsedSince()); Assert.assertEquals(loadedFromJava, loadedFromProto); final SortedSet<Diff> diff = new AtlasDelta(loadedFromJava, loadedFromProto, true) .generate().getDifferences(); Assert.assertTrue(diff.isEmpty()); }
final ByteArrayResource resourceWithRelation = new ByteArrayResource(1024 * 1024 * 1); final ByteArrayResource resourceNoRelation = new ByteArrayResource(1024 * 1024 * 1);
final ByteArrayResource resourceWithRelation = new ByteArrayResource(1024 * 1024 * 1); final ByteArrayResource resourceNoRelation = new ByteArrayResource(1024 * 1024 * 1);
@Test public void testPointsForMissingRelation() { final PackedAtlasBuilder builderWithRelation = new PackedAtlasBuilder(); builderWithRelation.addPoint(1L, Location.forString("1,1"), Maps.hashMap("somePoint", "inTwoAtlases")); final RelationBean bean = new RelationBean(); bean.addItem(1L, "Point appears in multiple atlases, but parent relation is only in one atlas", ItemType.POINT); builderWithRelation.addRelation(2, 2, bean, Maps.hashMap()); final PackedAtlasBuilder builderNoRelation = new PackedAtlasBuilder(); builderNoRelation.addPoint(1L, Location.forString("1,1"), Maps.hashMap("somePoiont", "inTwoAtlases")); final PackedAtlas atlasWithRelation = (PackedAtlas) builderWithRelation.get(); final PackedAtlas atlasNoRelation = (PackedAtlas) builderNoRelation.get(); // 1 MiB resources final ByteArrayResource resourceWithRelation = new ByteArrayResource(1024 * 1024 * 1); final ByteArrayResource resourceNoRelation = new ByteArrayResource(1024 * 1024 * 1); atlasWithRelation.save(resourceWithRelation); atlasNoRelation.save(resourceNoRelation); /* * Add the Atlas that is missing the relation second so that it overwrites the first one. * See the MultiAtlas.populateReferences() function to see this overwrite happen. */ final Atlas multiAtlas = new AtlasResourceLoader().load(resourceWithRelation, resourceNoRelation); Assert.assertTrue(!multiAtlas.point(1L).relations().isEmpty()); }
final ByteArrayResource resourceWithRelation = new ByteArrayResource(1024 * 1024 * 1); final ByteArrayResource resourceNoRelation = new ByteArrayResource(1024 * 1024 * 1);
@Test public void testZipWritableResource() { final ByteArrayResource source = new ByteArrayResource(); final ZipWritableResource writable = new ZipWritableResource(source); writable.writeAndClose(new StringResource("byte! " + CONTENTS_1).withName(NAME_1), new StringResource("byte! " + CONTENTS_2).withName(NAME_2)); final ZipResource resource = new ZipResource(source); int counter = 0; for (final Resource entry : resource.entries()) { final String name = entry.getName(); final String contents = entry.all(); logger.info(name + " -> " + contents); if (counter == 0) { Assert.assertEquals(NAME_1, name); Assert.assertEquals("byte! " + CONTENTS_1, contents); } else { Assert.assertEquals(NAME_2, name); Assert.assertEquals("byte! " + CONTENTS_2, contents); } counter++; } } }
@Test public void testAtlasFileExtensionFilter() { final PackedAtlasBuilder builder = new PackedAtlasBuilder(); builder.addPoint(1, new Location(Latitude.degrees(0), Longitude.degrees(0)), Maps.hashMap()); final Atlas atlas = builder.get(); final ByteArrayResource atlasResource = new ByteArrayResource() .withName("iDoNotHaveAnAtlasExt.txt"); atlas.save(atlasResource); Assert.assertNull(new AtlasResourceLoader().load(atlasResource)); Assert.assertNotNull(new AtlasResourceLoader().withAtlasFileExtensionFilterSetTo(false) .load(atlasResource)); }
final ByteArrayResource resourceWithRelation = new ByteArrayResource(1024 * 1024 * 1); final ByteArrayResource resourceNoRelation = new ByteArrayResource(1024 * 1024 * 1);