@Override public PartitionKey deserialize(JsonElement jsonElement, Type type, JsonDeserializationContext jsonDeserializationContext) throws JsonParseException { JsonObject jsonObject = jsonElement.getAsJsonObject(); PartitionKey.Builder builder = PartitionKey.builder(); for (Map.Entry<String, JsonElement> entry : jsonObject.entrySet()) { JsonArray jsonArray = entry.getValue().getAsJsonArray(); builder.addField(entry.getKey(), deserializeComparable(jsonArray, jsonDeserializationContext)); } return builder.build(); }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || !(o instanceof BasicPartition)) { return false; } BasicPartition that = (BasicPartition) o; return key.equals(that.key) && relativePath.equals(that.relativePath); }
@VisibleForTesting static long timeForPartitionKey(PartitionKey key) { int year = (Integer) key.getField(FIELD_YEAR); int month = (Integer) key.getField(FIELD_MONTH) - 1; int day = (Integer) key.getField(FIELD_DAY); int hour = (Integer) key.getField(FIELD_HOUR); int minute = (Integer) key.getField(FIELD_MINUTE); Calendar calendar = Calendar.getInstance(); calendar.clear(); //noinspection MagicConstant calendar.set(year, month, day, hour, minute); return calendar.getTimeInMillis(); }
Map<String, String> props = new HashMap<>(); props.put(CassandraConstants.CQL_VERSION, CassandraConstants.CQL_VERSION_3_0); emf = Persistence.createEntityManagerFactory("cass_pu", props); ProductionRecord pr = new ProductionRecord(); pr.setSolrQuery("some solr query"); pr.setAdditionColumn("col1"); ClusteringKey ck = new ClusteringKey(); ck.setClusteringKey(UUID.randomUUID()); PartitionKey pk = new PartitionKey(); pk.setKey1("k1"); pk.setKey2("k2"); pk.setKey3("k3"); ck.setKey(pk); pr.setClusteringkey(ck); em.persist(pr);
/** * Validates the partition key against the partitioning. */ private static void validatePartitionKey(PartitionKey key, Partitioning partitioning) { if (!partitioning.getFields().keySet().equals(key.getFields().keySet())) { throw new IllegalArgumentException(String.format( "Partition key is invalid: It contains fields %s, but the partitioning requires %s", key.getFields().keySet(), partitioning.getFields().keySet())); } for (Map.Entry<String, FieldType> entry : partitioning.getFields().entrySet()) { String fieldName = entry.getKey(); FieldType fieldType = entry.getValue(); Comparable fieldValue = key.getField(fieldName); if (fieldValue == null) { throw new IllegalArgumentException( String.format("Incomplete partition key: value for field '%s' is missing", fieldName)); } try { fieldType.validate(fieldValue); } catch (IllegalArgumentException e) { throw new IllegalArgumentException(String.format( "Invalid partition key: Value for field '%s' is incompatible with the partitioning: %s", fieldName, e.getMessage())); } } }
@Test public void testBuilderGetter() { PartitionKey key = PartitionKey.builder() .addField("a", "value") .addField("b", 1L) .addField("c", -17) .addField("d", true) .addIntField("e", 42) .addLongField("f", 15) .addStringField("g", "ghijk") .build(); Assert.assertEquals("value", key.getField("a")); Assert.assertEquals(1L, key.getField("b")); Assert.assertEquals(-17, key.getField("c")); Assert.assertEquals(true, key.getField("d")); Assert.assertEquals(42, key.getField("e")); Assert.assertEquals(15L, key.getField("f")); Assert.assertEquals("ghijk", key.getField("g")); }
@Test public void testEqualityHashCode() { PartitionKey key1 = PartitionKey.builder() .addField("a", "value") .addField("b", 1L) .addField("c", -17) .build(); PartitionKey key2 = PartitionKey.builder() .addField("b", 1L) .addField("c", -17) .addField("a", "value") .build(); Assert.assertEquals(key1, key2); Assert.assertEquals(key1.hashCode(), key2.hashCode()); } }
public PartitionAlreadyExistsException(String datasetName, PartitionKey partitionKey) { super(String.format("Dataset '%s' already has a partition with the same key: %s", datasetName, partitionKey.toString())); } }
@Override public int hashCode() { int result = partitionKey.hashCode(); result = 31 * result + processState.hashCode(); result = 31 * result + numFailures; result = 31 * result + (int) (timestamp ^ (timestamp >>> 32)); return result; } }
/** * Create the partition key. * * @throws IllegalStateException if no fields have been added, * or the partitioning is known and not all fields have been added. */ public PartitionKey build() { if (fields.isEmpty()) { throw new IllegalStateException("Partition key cannot be empty."); } if (partitioning != null && !partitioning.getFields().keySet().equals(fields.keySet())) { throw new IllegalStateException(String.format( "Partition key is incomplete: It only contains fields %s, but the partitioning requires %s", fields.keySet(), partitioning.getFields().keySet())); } return new PartitionKey(fields); } }
for (int i = 0; i < 4; i++) { for (int l = 0; l < 4; l++) { final PartitionKey key = PartitionKey.builder() .addField("s", String.format("%c-%d", 'a' + s, s)) .addField("i", i * 100) boolean success = testFilter(dataset, allPartitionDetails, PartitionFilter.builder() .addValueCondition("l", key.getField("l")) .addValueCondition("s", key.getField("s")) .addValueCondition("i", key.getField("i")) .build());
private void dropPartitionFromExplore(PartitionKey key) { if (exploreEnabled) { ExploreFacade exploreFacade = exploreFacadeProvider.get(); if (exploreFacade != null) { try { exploreFacade.dropPartition(datasetInstanceId, spec, key); } catch (Exception e) { throw new DataSetException(String.format( "Unable to drop partition for key %s from explore table.", key.toString()), e); } } } }
/** * Validates the partition key against the partitioning. */ private static void validatePartitionKey(PartitionKey key, Partitioning partitioning) { if (!partitioning.getFields().keySet().equals(key.getFields().keySet())) { throw new IllegalArgumentException(String.format( "Partition key is invalid: It contains fields %s, but the partitioning requires %s", key.getFields().keySet(), partitioning.getFields().keySet())); } for (Map.Entry<String, FieldType> entry : partitioning.getFields().entrySet()) { String fieldName = entry.getKey(); FieldType fieldType = entry.getValue(); Comparable fieldValue = key.getField(fieldName); if (fieldValue == null) { throw new IllegalArgumentException( String.format("Incomplete partition key: value for field '%s' is missing", fieldName)); } try { fieldType.validate(fieldValue); } catch (IllegalArgumentException e) { throw new IllegalArgumentException(String.format( "Invalid partition key: Value for field '%s' is incompatible with the partitioning: %s", fieldName, e.getMessage())); } } }
/** * Set the partition key of the output partition when using PartitionedFileSet as an OutputFormatProvider. * This key is used as the partition key for the new file, and also to generate an output file path - if that path * is not explicitly given as an argument itself. * * @param key the partition key * @param arguments the runtime arguments for a partitioned dataset */ public static void setOutputPartitionKey(Map<String, String> arguments, PartitionKey key) { for (Map.Entry<String, ? extends Comparable> entry : key.getFields().entrySet()) { arguments.put(OUTPUT_PARTITION_KEY_PREFIX + entry.getKey(), entry.getValue().toString()); } }
public Map<String, String> getOutputArguments(long snapshotTime, Map<String, String> otherProperties) { Map<String, String> args = new HashMap<>(); args.putAll(otherProperties); PartitionKey outputKey = PartitionKey.builder().addLongField(SNAPSHOT_FIELD, snapshotTime).build(); PartitionedFileSetArguments.setOutputPartitionKey(args, outputKey); return args; }
@VisibleForTesting static long timeForPartitionKey(PartitionKey key) { int year = (Integer) key.getField(FIELD_YEAR); int month = (Integer) key.getField(FIELD_MONTH) - 1; int day = (Integer) key.getField(FIELD_DAY); int hour = (Integer) key.getField(FIELD_HOUR); int minute = (Integer) key.getField(FIELD_MINUTE); Calendar calendar = Calendar.getInstance(); calendar.clear(); //noinspection MagicConstant calendar.set(year, month, day, hour, minute); return calendar.getTimeInMillis(); }
private void dropPartitionFromExplore(PartitionKey key) { if (exploreEnabled) { ExploreFacade exploreFacade = exploreFacadeProvider.get(); if (exploreFacade != null) { try { exploreFacade.dropPartition(datasetInstanceId, spec, key); } catch (Exception e) { throw new DataSetException(String.format( "Unable to drop partition for key %s from explore table.", key.toString()), e); } } } }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || !(o instanceof BasicPartition)) { return false; } BasicPartition that = (BasicPartition) o; return key.equals(that.key) && relativePath.equals(that.relativePath); }
@Override public JsonElement serialize(PartitionKey partitionKey, Type type, JsonSerializationContext jsonSerializationContext) { JsonObject jsonObj = new JsonObject(); for (Map.Entry<String, Comparable> entry : partitionKey.getFields().entrySet()) { jsonObj.add(entry.getKey(), serializeComparable(entry.getValue(), jsonSerializationContext)); } return jsonObj; } }