/** * Creates and returns a File pointing to the directory for the given store and task, given a particular base directory. * * @param storeBaseDir the base directory to use * @param storeName the store name to use * @param taskName the task name which is referencing the store * @return the partition directory for the store */ public static File getStorePartitionDir(File storeBaseDir, String storeName, TaskName taskName) { return new File(storeBaseDir, (storeName + File.separator + taskName.toString()).replace(' ', '_')); } }
@VisibleForTesting File getStoreLocation(String storeName) { return new File(storeBaseDir, (storeName + File.separator + taskName.toString()).replace(' ', '_')); }
@VisibleForTesting File getStoreLocation(String storeName) { return new File(storeBaseDir, (storeName + File.separator + taskName.toString()).replace(' ', '_')); }
@VisibleForTesting File getStoreLocation(String storeName) { return new File(storeBaseDir, (storeName + File.separator + taskName.toString()).replace(' ', '_')); }
@VisibleForTesting File getStoreLocation(String storeName) { return new File(storeBaseDir, (storeName + File.separator + taskName.toString()).replace(' ', '_')); }
@VisibleForTesting File getStoreLocation(String storeName) { return new File(storeBaseDir, (storeName + File.separator + taskName.toString()).replace(' ', '_')); }
@Override public void serialize(TaskName taskName, JsonGenerator jsonGenerator, SerializerProvider provider) throws IOException, JsonProcessingException { jsonGenerator.writeObject(taskName.toString()); } }
@Override public void serialize(TaskName taskName, JsonGenerator jsonGenerator, SerializerProvider provider) throws IOException, JsonProcessingException { jsonGenerator.writeObject(taskName.toString()); } }
@Override public void serialize(TaskName taskName, JsonGenerator jsonGenerator, SerializerProvider provider) throws IOException, JsonProcessingException { jsonGenerator.writeObject(taskName.toString()); } }
@Override public void serialize(TaskName taskName, JsonGenerator jsonGenerator, SerializerProvider provider) throws IOException, JsonProcessingException { jsonGenerator.writeObject(taskName.toString()); } }
@Override public void serialize(TaskName taskName, JsonGenerator jsonGenerator, SerializerProvider provider) throws IOException, JsonProcessingException { jsonGenerator.writeObject(taskName.toString()); } }
@Override public byte[] toBytes(KafkaCheckpointLogKey key) { try { return mapper.writeValueAsBytes(ImmutableMap.of( SSP_GROUPER_FACTORY_FIELD, key.getGrouperFactoryClassName(), TASK_NAME_FIELD, key.getTaskName().toString(), TYPE_FIELD, key.getType() )); } catch (Exception e) { throw new SamzaException(String.format("Exception in serializing: %s", key), e); } }
@Override public byte[] toBytes(KafkaCheckpointLogKey key) { try { return mapper.writeValueAsBytes(ImmutableMap.of( SSP_GROUPER_FACTORY_FIELD, key.getGrouperFactoryClassName(), TASK_NAME_FIELD, key.getTaskName().toString(), TYPE_FIELD, key.getType() )); } catch (Exception e) { throw new SamzaException(String.format("Exception in serializing: %s", key), e); } }
@Override public byte[] toBytes(KafkaCheckpointLogKey key) { try { return mapper.writeValueAsBytes(ImmutableMap.of( SSP_GROUPER_FACTORY_FIELD, key.getGrouperFactoryClassName(), TASK_NAME_FIELD, key.getTaskName().toString(), TYPE_FIELD, key.getType() )); } catch (Exception e) { throw new SamzaException(String.format("Exception in serializing: %s", key), e); } }
@Override public void clearCheckpoints() { LOG.debug("Clearing all checkpoints in Azure table"); for (TaskName taskName : taskNames) { String partitionQueryKey = taskName.toString(); // Generate table query String partitionFilter = TableQuery.generateFilterCondition( PARTITION_KEY, TableQuery.QueryComparisons.EQUAL, partitionQueryKey); TableQuery<TaskCheckpointEntity> partitionQuery = TableQuery.from(TaskCheckpointEntity.class) .where(partitionFilter); // All entities in a given batch must have the same partition key deleteEntities(cloudTable.execute(partitionQuery).iterator()); } }
@Override public Checkpoint readLastCheckpoint(TaskName taskName) { if (!taskNames.contains(taskName)) { throw new SamzaException("reading checkpoint of unregistered/unwritten task"); } // Create the query for taskName String partitionQueryKey = taskName.toString(); String partitionFilter = TableQuery.generateFilterCondition( PARTITION_KEY, TableQuery.QueryComparisons.EQUAL, partitionQueryKey); TableQuery<TaskCheckpointEntity> query = TableQuery.from(TaskCheckpointEntity.class).where(partitionFilter); ImmutableMap.Builder<SystemStreamPartition, String> builder = ImmutableMap.builder(); try { for (TaskCheckpointEntity taskCheckpointEntity : cloudTable.execute(query)) { // Recreate the SSP offset String serializedSSP = taskCheckpointEntity.getRowKey(); builder.put(deserializeSystemStreamPartition(serializedSSP), taskCheckpointEntity.getOffset()); } } catch (NoSuchElementException e) { LOG.warn("No checkpoints found found for registered taskName={}", taskName); // Return null if not entity elements are not found return null; } LOG.debug("Received checkpoint state for taskName=%s", taskName); return new Checkpoint(builder.build()); }
String offset = entry.getValue(); String partitionKey = taskName.toString(); checkValidKey(partitionKey, "Taskname"); String rowKey = serializeSystemStreamPartition(ssp);