public void storeSchema(ResourceSchema schema, String location, Job job) throws IOException { // Store the schema in a side file in the same directory. MapReduce // does not include files starting with "_" when reading data for a job. JsonMetadata metadataWriter = new JsonMetadata(); byte recordDel = '\n'; byte fieldDel = '\t'; metadataWriter.setFieldDel(fieldDel); metadataWriter.setRecordDel(recordDel); metadataWriter.storeSchema(schema, location, job); }
/** * For JsonMetadata schema is considered optional * This method suppresses (and logs) errors if they are encountered. * */ @Override public ResourceSchema getSchema(String location, Job job) throws IOException { return getSchema(location, job, false); }
@Override public ResourceSchema getSchema(String location, Job job) throws IOException { ResourceSchema s; if (schema!=null) { s = schema; } else { // Parse the schema s = (new JsonMetadata()).getSchema(location, job, true); if (s == null) { throw new IOException("Unable to parse schema found in file in " + location); } } // Now that we have determined the schema, store it in our // UDFContext properties object so we have it when we need it on the // backend UDFContext udfc = UDFContext.getUDFContext(); Properties p = udfc.getUDFProperties(this.getClass(), new String[]{udfcSignature}); p.setProperty(SCHEMA_SIGNATURE, s.toString()); return s; }
Set<ElementDescriptor> schemaFileSet = null; try { schemaFileSet = findMetaFile(location, schemaFileName, conf); } catch (IOException e) { String msg = "Could not find schema file for "+ location; return nullOrException(isSchemaOn, msg, e); } else { String msg = "Could not find schema file for "+location; return nullOrException(isSchemaOn, msg, null); } catch (JsonParseException e) { String msg = "Unable to load Resource Schema for "+location; return nullOrException(isSchemaOn, msg, e); } catch (JsonMappingException e) { String msg = "Unable to load Resource Schema for "+location; return nullOrException(isSchemaOn, msg, e); } catch (IOException e) { String msg = "Unable to load Resource Schema for "+location; return nullOrException(isSchemaOn, msg, e);
Set<ElementDescriptor> statFileSet = null; try { statFileSet = findMetaFile(location, statFileName, conf); } catch (IOException e) { log.warn("could not fine stat file for " + location);
@Override public ResourceSchema getSchema(String location, Job job) throws IOException { if (!dontLoadSchema) { schema = (new JsonMetadata()).getSchema(location, job, isSchemaOn); if (signature != null && schema != null) { if(tagFile) { schema = Utils.getSchemaWithInputSourceTag(schema, "INPUT_FILE_NAME"); } else if(tagPath) { schema = Utils.getSchemaWithInputSourceTag(schema, "INPUT_FILE_PATH"); } Properties p = UDFContext.getUDFContext().getUDFProperties(this.getClass(), new String[] {signature}); p.setProperty(signature + ".schema", schema.toString()); } } return schema; }
@Override public void storeSchema(ResourceSchema schema, String location, Job job) throws IOException { if (isSchemaOn) { JsonMetadata metadataWriter = new JsonMetadata(); byte recordDel = '\n'; metadataWriter.setFieldDel(fieldDel); metadataWriter.setRecordDel(recordDel); metadataWriter.storeSchema(schema, location, job); } }