public AggregateGafferElements(final Schema gafferSchema) { jsonGafferSchema = gafferSchema.toCompactJson(); }
public ExtractKeyFromElements(final Schema gafferSchema) { jsonGafferSchema = gafferSchema.toCompactJson(); }
public WriteUnsortedDataFunction(final String tempFilesDir, final SchemaUtils schemaUtils, final Map<String, Map<Object, Integer>> groupToSplitPoints) { this.tempFilesDir = tempFilesDir; this.gafferSchema = schemaUtils.getGafferSchema().toCompactJson(); this.groupToSplitPoints = groupToSplitPoints; }
public IteratorSettingBuilder schema(final Schema schema) { try { setting.addOption(AccumuloStoreConstants.SCHEMA, new String(schema.toCompactJson(), CommonConstants.UTF_8)); } catch (final UnsupportedEncodingException e) { throw new SchemaException("Unable to deserialise schema from JSON", e); } return this; }
@Override public JobConf createJobConf(final SampleDataForSplitPoints operation, final String mapperGeneratorClassName, final Store store) throws IOException { final JobConf jobConf = new JobConf(new Configuration()); LOGGER.info("Setting up job conf"); jobConf.set(SCHEMA, new String(store.getSchema().toCompactJson(), CommonConstants.UTF_8)); LOGGER.info("Added {} {} to job conf", SCHEMA, new String(store.getSchema().toCompactJson(), CommonConstants.UTF_8)); jobConf.set(MAPPER_GENERATOR, mapperGeneratorClassName); LOGGER.info("Added {} of {} to job conf", MAPPER_GENERATOR, mapperGeneratorClassName); jobConf.set(VALIDATE, String.valueOf(operation.isValidate())); LOGGER.info("Added {} option of {} to job conf", VALIDATE, operation.isValidate()); jobConf.set(PROPORTION_TO_SAMPLE, String.valueOf(operation.getProportionToSample())); LOGGER.info("Added {} option of {} to job conf", PROPORTION_TO_SAMPLE, String.valueOf(operation.getProportionToSample())); final Integer numTasks = operation.getNumMapTasks(); if (null != numTasks) { jobConf.setNumMapTasks(numTasks); LOGGER.info("Set number of map tasks to {} on job conf", numTasks); } jobConf.setNumReduceTasks(1); LOGGER.info("Set number of reduce tasks to 1 on job conf"); jobConf.set(AccumuloStoreConstants.ACCUMULO_ELEMENT_CONVERTER_CLASS, ((AccumuloStore) store).getKeyPackage().getKeyConverter().getClass().getName()); return jobConf; }
@Override public JobConf createJobConf(final AddElementsFromHdfs operation, final String mapperGeneratorClassName, final Store store) throws IOException { final JobConf jobConf = new JobConf(new Configuration()); LOGGER.info("Setting up job conf"); jobConf.set(SCHEMA, new String(store.getSchema().toCompactJson(), CommonConstants.UTF_8)); LOGGER.debug("Added {} {} to job conf", SCHEMA, new String(store.getSchema().toCompactJson(), CommonConstants.UTF_8)); jobConf.set(MAPPER_GENERATOR, mapperGeneratorClassName); LOGGER.info("Added {} of {} to job conf", MAPPER_GENERATOR, mapperGeneratorClassName); jobConf.set(VALIDATE, String.valueOf(operation.isValidate())); LOGGER.info("Added {} option of {} to job conf", VALIDATE, operation.isValidate()); if (null != operation.getNumMapTasks()) { jobConf.setNumMapTasks(operation.getNumMapTasks()); LOGGER.info("Set number of map tasks to {} on job conf", operation.getNumMapTasks()); } if (null != operation.getNumReduceTasks()) { jobConf.setNumReduceTasks(operation.getNumReduceTasks()); LOGGER.info("Set number of reduce tasks to {} on job conf", operation.getNumReduceTasks()); } jobConf.set(AccumuloStoreConstants.ACCUMULO_ELEMENT_CONVERTER_CLASS, ((AccumuloStore) store).getKeyPackage().getKeyConverter().getClass().getName()); return jobConf; }
public RetrieveElementsFromFile(final Path filePath, final FilterPredicate filter, final Schema gafferSchema, final ConcurrentLinkedQueue<Element> queue, final boolean needsValidatorsAndFiltersApplying, final boolean skipValidation, final View view, final User user) { this.filePath = filePath; this.filter = filter; this.jsonGafferSchema = gafferSchema.toCompactJson(); this.gafferSchema = gafferSchema; if (gafferSchema.getVisibilityProperty() != null) { this.visibility = gafferSchema.getVisibilityProperty(); } else { this.visibility = new String(); } if (user != null && user.getDataAuths() != null) { final Set<String> dataAuths = user.getDataAuths(); this.auths = new Authorisations(dataAuths.toArray(new String[dataAuths.size()])); } else { this.auths = new Authorisations(); } this.queue = queue; this.view = view; this.needsValidatorsAndFiltersApplying = needsValidatorsAndFiltersApplying; this.skipValidation = skipValidation; if (filePath.getName().contains("=")) { group = filePath.getName().split("=")[1]; } else { group = filePath.getParent().getName().split("=")[1]; } elementDefinitionJson = view.getElement(group).toCompactJson(); }
public <OP extends Validatable & Operation> GafferAdder(final OP operation, final Store store) { this.store = store; this.validate = operation.isValidate(); this.skipInvalid = operation.isSkipInvalidElements(); final String maxQueueSizeOption = operation.getOption(FlinkConstants.MAX_QUEUE_SIZE); this.maxQueueSize = null != maxQueueSizeOption ? Integer.parseInt(maxQueueSizeOption) : MAX_QUEUE_SIZE_DEFAULT; graphId = store.getGraphId(); schema = store.getSchema().toCompactJson(); properties = store.getProperties().getProperties(); }
conf.set(ElementInputFormat.SCHEMA, new String(getSchema().toCompactJson(), CommonConstants.UTF_8)); LOGGER.debug("Updating configuration with Schema of {}", getSchema()); conf.set(ElementInputFormat.VIEW, new String(view.toCompactJson(), CommonConstants.UTF_8));