@JsonIgnore public Schema getDeserialisedSchema() { if (null == deserialisedSchema) { if (null == graph) { deserialisedSchema = null != schema ? Schema.fromJson(schema) : null; } else { deserialisedSchema = graph.getSchema(); } } return deserialisedSchema; }
public Builder schema(final InputStream schema) { return schema(Schema.fromJson(schema)); }
@Override protected void setup(final Context context) { doValidation = Boolean.parseBoolean(context.getConfiguration().get(VALIDATE)); try { schema = Schema.fromJson(context.getConfiguration().get(SCHEMA).getBytes(CommonConstants.UTF_8)); } catch (final UnsupportedEncodingException e) { throw new RuntimeException(e.getMessage(), e); } elementValidator = new ElementValidator(schema); final String generatorClass = context.getConfiguration().get(MAPPER_GENERATOR); try { mapperGenerator = Class.forName(SimpleClassNameIdResolver.getClassName(generatorClass)).asSubclass(MapperGenerator.class).newInstance(); } catch (final ClassNotFoundException | InstantiationException | IllegalAccessException e) { throw new IllegalArgumentException("Element generator could be created: " + generatorClass, e); } }
@Override protected void setup(final Context context) { final String schemaJson = context.getConfiguration().get(SCHEMA); if (null != schemaJson) { schema = Schema.fromJson(StringUtil.toBytes(schemaJson)); } }
@Override public RecordReader<Element, NullWritable> createRecordReader(final InputSplit split, final TaskAttemptContext context) throws IOException, InterruptedException { log.setLevel(getLogLevel(context)); final Configuration conf = context.getConfiguration(); final String keyPackageClass = conf.get(KEY_PACKAGE); final Schema schema = Schema.fromJson(conf.get(SCHEMA).getBytes(CommonConstants.UTF_8)); final View view = View.fromJson(conf.get(VIEW).getBytes(CommonConstants.UTF_8)); try { return new ElementWithPropertiesRecordReader(keyPackageClass, schema, view); } catch (final StoreException | SchemaException | SerialisationException e) { throw new IOException("Exception creating RecordReader", e); } }
public void initialise() { if (null == store) { store = Store.createStore(graphId, Schema.fromJson(schema), StoreProperties.loadStoreProperties(properties)); } }
public EntryToElement(final byte[] serialisedConf) { try { final Configuration conf = Utils.deserialiseConfiguration(serialisedConf); final String keyPackageClass = conf.get(KEY_PACKAGE); final Schema schema = Schema.fromJson(conf.get(SCHEMA).getBytes(CommonConstants.UTF_8)); final AccumuloKeyPackage keyPackage = Class .forName(keyPackageClass) .asSubclass(AccumuloKeyPackage.class) .newInstance(); keyPackage.setSchema(schema); converter = keyPackage.getKeyConverter(); LOGGER.info("Initialised EntryToElement with AccumuloElementConverter of {}", converter.getClass().getName()); view = View.fromJson(conf.get(AbstractGetRDDHandler.VIEW).getBytes(CommonConstants.UTF_8)); LOGGER.info("Initialised EntryToElement with View of {}", view.toString()); } catch (final InstantiationException | IllegalAccessException | ClassNotFoundException | IOException e) { throw new RuntimeException("Exception creating AccumuloKeyPackage from Configuration", e); } }
@Override protected void setup(final Context context) { super.setup(context); proportionToSample = context.getConfiguration().getFloat(AccumuloSampleDataForSplitPointsJobFactory.PROPORTION_TO_SAMPLE, 0.001f); final Schema schema; try { schema = Schema.fromJson(context.getConfiguration() .get(AccumuloSampleDataForSplitPointsJobFactory.SCHEMA).getBytes(CommonConstants.UTF_8)); } catch (final UnsupportedEncodingException e) { throw new SchemaException("Unable to deserialise Store Schema from JSON", e); } try { elementConverter = Class .forName(context.getConfiguration().get(AccumuloStoreConstants.ACCUMULO_ELEMENT_CONVERTER_CLASS)) .asSubclass(AccumuloElementConverter.class) .getConstructor(Schema.class) .newInstance(schema); } catch (final ClassNotFoundException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) { throw new IllegalArgumentException("Element converter could not be created: " + context.getConfiguration().get(AccumuloStoreConstants.ACCUMULO_ELEMENT_CONVERTER_CLASS), e); } }
@Override public void init(final SortedKeyValueIterator<Key, Value> source, final Map<String, String> options, final IteratorEnvironment env) throws IOException { super.init(source, options, env); try { schema = Schema.fromJson(options.get(AccumuloStoreConstants.SCHEMA).getBytes(CommonConstants.UTF_8)); } catch (final UnsupportedEncodingException e) { throw new SchemaException("Unable to deserialise the schema from json", e); } LOGGER.debug("Initialising AggregatorIterator with schema {}", schema); final String elementConverterClass = options.get(AccumuloStoreConstants.ACCUMULO_ELEMENT_CONVERTER_CLASS); try { elementConverter = Class .forName(elementConverterClass) .asSubclass(AccumuloElementConverter.class) .getConstructor(Schema.class) .newInstance(schema); LOGGER.debug("Creating AccumuloElementConverter of class {}", elementConverterClass); } catch (final ClassNotFoundException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) { throw new AggregationException("Failed to create element converter of the class name provided (" + elementConverterClass + ")", e); } }
@Override public void call(final Iterator<Element> elements) throws Exception { SchemaUtils utils = new SchemaUtils(Schema.fromJson(gafferSchema)); final WriteUnsortedData writer = new WriteUnsortedData(tempFilesDir, utils, groupToSplitPoints); try { writer.writeElements(elements); } catch (final OperationException e) { LOGGER.error("Failed to write partition: {}", e); } } }
@Override public void init(final SortedKeyValueIterator<Key, Value> source, final Map<String, String> options, final IteratorEnvironment env) throws IOException { super.init(source, options, env); this.source = source; try { schema = Schema.fromJson(options.get(AccumuloStoreConstants.SCHEMA).getBytes(CommonConstants.UTF_8)); } catch (final UnsupportedEncodingException e) { throw new SchemaException("Unable to deserialise the schema", e); } LOGGER.debug("Initialising RowIDAggregator with schema {}", schema); final String elementConverterClass = options.get(AccumuloStoreConstants.ACCUMULO_ELEMENT_CONVERTER_CLASS); try { elementConverter = Class .forName(elementConverterClass) .asSubclass(AccumuloElementConverter.class) .getConstructor(Schema.class) .newInstance(schema); LOGGER.debug("Creating AccumuloElementConverter of class {}", elementConverterClass); } catch (final ClassNotFoundException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) { throw new AggregationException("Failed to create element converter of the class name provided (" + elementConverterClass + ")", e); } group = options.get(AccumuloStoreConstants.COLUMN_FAMILY); LOGGER.debug("group is set to {}", group); aggregator = schema.getElement(group).getFullAggregator(); LOGGER.debug("Aggregator is set to {}", aggregator); }
super.init(source, options, env); try { schema = Schema.fromJson(options.get(AccumuloStoreConstants.SCHEMA).getBytes(CommonConstants.UTF_8)); } catch (final UnsupportedEncodingException e) { throw new SchemaException("Unable to deserialise the schema", e);
public static void main(final String[] args) { if (args.length != 4) { System.err.println("Usage: " + RoadTrafficDataLoader.class.getSimpleName() + " <graphConfigFile> <schemaDir> <storePropsFile> <roadTrafficDataFile.csv>"); System.exit(1); } final String graphConfigFile = args[0]; final String schemaDir = args[1]; final String storePropertiesFile = args[2]; final String dataFile = args[3]; final GraphConfig config = new GraphConfig.Builder().json(new File(graphConfigFile).toPath()).build(); final Schema schema = Schema.fromJson(new File(schemaDir).toPath()); final StoreProperties storeProperties = StoreProperties.loadStoreProperties(storePropertiesFile); final Graph graph = new Graph.Builder() .config(config) .addSchemas(schema) .storeProperties(storeProperties) .build(); final User user = new User(); final RoadTrafficDataLoader dataLoader = new RoadTrafficDataLoader(graph, user); LOGGER.info("Loading data"); try { dataLoader.load(new File(dataFile)); LOGGER.info("Data has been loaded"); } catch (final Exception e) { LOGGER.info("Unable to load data: " + e.getMessage()); throw new RuntimeException("Unable to load data", e); } }
@Override public Tuple2<List<Object>, Element> call(final Element element) throws Exception { if (null == gafferSchema) { gafferSchema = Schema.fromJson(jsonGafferSchema); } final String group = element.getGroup(); final List<Object> list = new ArrayList<>(); list.add(group); if (gafferSchema.getEntityGroups().contains(group)) { final Entity entity = (Entity) element; list.add(entity.getVertex()); } else { final Edge edge = (Edge) element; list.add(edge.getSource()); list.add(edge.getDestination()); list.add(edge.getDirectedType()); } for (final String property : gafferSchema.getElement(group).getGroupBy()) { list.add(element.getProperty(property)); } return new Tuple2<>(list, element); } }
@Override public Element call(final Element v1, final Element v2) { if (null == gafferSchema) { gafferSchema = Schema.fromJson(jsonGafferSchema); } final ElementAggregator aggregator = gafferSchema.getElement(v2.getGroup()).getIngestAggregator(); return aggregator.apply(v1, v2); } }
final IteratorEnvironment env) throws IOException { super.init(source, options, env); schema = Schema.fromJson(StringUtil.toBytes(options.get(AccumuloStoreConstants.SCHEMA))); LOGGER.debug("Initialising AbstractElementFilter with Schema {}", schema);
final Schema schema = Schema.fromJson(getSchemaPaths(args));
schemaUtils = new SchemaUtils(Schema.fromJson(jsonGafferSchema));
public void simpleExportWithCustomGraph() { // --------------------------------------------------------- final Schema schema = Schema.fromJson(StreamUtil.openStreams(getClass(), "operation/schema")); final StoreProperties storeProperties = StoreProperties.loadStoreProperties(StreamUtil.openStream(getClass(), "othermockaccumulostore.properties")); final OperationChain<Iterable<? extends Element>> opChain = new OperationChain.Builder() .first(new GetAllElements.Builder() .view(new View.Builder() .edge("edge") .build()) .build()) .then(new ExportToOtherGraph.Builder() .graphId("newGraphId") .schema(schema) .storeProperties(storeProperties) .build()) .build(); // --------------------------------------------------------- showExample(opChain, "This example will export all Edges with group 'edge' to another Gaffer graph with new ID 'newGraphId'. " + "The new graph will have the custom provided schema (note it must contain the same Edge group 'edge' otherwise the exported edges will be invalid') and custom store properties. " + "The store properties could be any store properties e.g. Accumulo, HBase, Map, Proxy store properties."); }