@Override public Schema determineSchemaOrReturnErrorSchema(Configuration conf, Properties props) { // AvroSerDe does not propagate initialization exceptions. Instead, it stores just an exception's message in // this.configErrors (see https://issues.apache.org/jira/browse/HIVE-7868). In Presto, such behavior is not // at all useful, as silenced exception usually carries important information which may be otherwise unavailable. try { return AvroSerdeUtils.determineSchemaOrThrowException(conf, props); } catch (IOException | AvroSerdeException e) { throw new RuntimeException(e); } } }
/** * Attempt to determine the schema via the usual means, but do not throw * an exception if we fail. Instead, signal failure via a special * schema. This is used because Hive calls init on the serde during * any call, including calls to update the serde properties, meaning * if the serde is in a bad state, there is no way to update that state. */ public Schema determineSchemaOrReturnErrorSchema(Configuration conf, Properties props) { try { configErrors = ""; return AvroSerdeUtils.determineSchemaOrThrowException(conf, props); } catch(AvroSerdeException he) { LOG.warn("Encountered AvroSerdeException determining schema. Returning " + "signal schema to indicate problem", he); configErrors = new String("Encountered AvroSerdeException determining schema. Returning " + "signal schema to indicate problem: " + he.getMessage()); return schema = SchemaResolutionProblem.SIGNAL_BAD_SCHEMA; } catch (Exception e) { LOG.warn("Encountered exception determining schema. Returning signal " + "schema to indicate problem", e); configErrors = new String("Encountered exception determining schema. Returning signal " + "schema to indicate problem: " + e.getMessage()); return SchemaResolutionProblem.SIGNAL_BAD_SCHEMA; } }
@Override public org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter getHiveRecordWriter(JobConf jobConf, Path path, Class<? extends Writable> valueClass, boolean isCompressed, Properties properties, Progressable progressable) throws IOException { Schema schema; try { schema = AvroSerdeUtils.determineSchemaOrThrowException(jobConf, properties); } catch (AvroSerdeException e) { throw new IOException(e); } GenericDatumWriter<GenericRecord> gdw = new GenericDatumWriter<GenericRecord>(schema); DataFileWriter<GenericRecord> dfw = new DataFileWriter<GenericRecord>(gdw); if (isCompressed) { int level = jobConf.getInt(DEFLATE_LEVEL_KEY, DEFAULT_DEFLATE_LEVEL); String codecName = jobConf.get(OUTPUT_CODEC, DEFLATE_CODEC); CodecFactory factory = codecName.equals(DEFLATE_CODEC) ? CodecFactory.deflateCodec(level) : CodecFactory.fromString(codecName); dfw.setCodec(factory); } dfw.create(schema, path.getFileSystem(jobConf).create(path)); return new AvroGenericRecordWriter(dfw); }
return AvroSerdeUtils.determineSchemaOrThrowException(job, props);
@Override public org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter getHiveRecordWriter(JobConf jobConf, Path path, Class<? extends Writable> valueClass, boolean isCompressed, Properties properties, Progressable progressable) throws IOException { Schema schema; try { schema = AvroSerdeUtils.determineSchemaOrThrowException(jobConf, properties); } catch (AvroSerdeException e) { throw new IOException(e); } GenericDatumWriter<GenericRecord> gdw = new GenericDatumWriter<GenericRecord>(schema); DataFileWriter<GenericRecord> dfw = new DataFileWriter<GenericRecord>(gdw); if (isCompressed) { int level = jobConf.getInt(DEFLATE_LEVEL_KEY, DEFAULT_DEFLATE_LEVEL); String codecName = jobConf.get(OUTPUT_CODEC, DEFLATE_CODEC); CodecFactory factory = codecName.equals(DEFLATE_CODEC) ? CodecFactory.deflateCodec(level) : CodecFactory.fromString(codecName); dfw.setCodec(factory); } dfw.create(schema, path.getFileSystem(jobConf).create(path)); return new AvroGenericRecordWriter(dfw); }
return AvroSerdeUtils.determineSchemaOrThrowException(job, props);
@Override public Schema determineSchemaOrReturnErrorSchema(Configuration conf, Properties props) { // AvroSerDe does not propagate initialization exceptions. Instead, it stores just an exception's message in // this.configErrors (see https://issues.apache.org/jira/browse/HIVE-7868). In Presto, such behavior is not // at all useful, as silenced exception usually carries important information which may be otherwise unavailable. try { return AvroSerdeUtils.determineSchemaOrThrowException(conf, props); } catch (IOException | AvroSerdeException e) { throw new RuntimeException(e); } } }
/** * Attempt to determine the schema via the usual means, but do not throw * an exception if we fail. Instead, signal failure via a special * schema. This is used because Hive calls init on the serde during * any call, including calls to update the serde properties, meaning * if the serde is in a bad state, there is no way to update that state. */ public static Schema determineSchemaOrReturnErrorSchema(Properties props) { try { return determineSchemaOrThrowException(props); } catch(AvroSerdeException he) { LOG.warn("Encountered AvroSerdeException determining schema. Returning " + "signal schema to indicate problem", he); return SchemaResolutionProblem.SIGNAL_BAD_SCHEMA; } catch (Exception e) { LOG.warn("Encountered exception determining schema. Returning signal " + "schema to indicate problem", e); return SchemaResolutionProblem.SIGNAL_BAD_SCHEMA; } } // Protected for testing and so we can pass in a conf for testing.
/** * Attempt to determine the schema via the usual means, but do not throw * an exception if we fail. Instead, signal failure via a special * schema. This is used because Hive calls init on the serde during * any call, including calls to update the serde properties, meaning * if the serde is in a bad state, there is no way to update that state. */ public Schema determineSchemaOrReturnErrorSchema(Configuration conf, Properties props) { try { configErrors = ""; return AvroSerdeUtils.determineSchemaOrThrowException(conf, props); } catch(AvroSerdeException he) { LOG.warn("Encountered AvroSerdeException determining schema. Returning " + "signal schema to indicate problem", he); configErrors = new String("Encountered AvroSerdeException determining schema. Returning " + "signal schema to indicate problem: " + he.getMessage()); return schema = SchemaResolutionProblem.SIGNAL_BAD_SCHEMA; } catch (Exception e) { LOG.warn("Encountered exception determining schema. Returning signal " + "schema to indicate problem", e); configErrors = new String("Encountered exception determining schema. Returning signal " + "schema to indicate problem: " + e.getMessage()); return SchemaResolutionProblem.SIGNAL_BAD_SCHEMA; } }
/** * Attempt to determine the schema via the usual means, but do not throw * an exception if we fail. Instead, signal failure via a special * schema. This is used because Hive calls init on the serde during * any call, including calls to update the serde properties, meaning * if the serde is in a bad state, there is no way to update that state. */ public Schema determineSchemaOrReturnErrorSchema(Configuration conf, Properties props) { try { configErrors = ""; return AvroSerdeUtils.determineSchemaOrThrowException(conf, props); } catch(AvroSerdeException he) { LOG.warn("Encountered AvroSerdeException determining schema. Returning " + "signal schema to indicate problem", he); configErrors = new String("Encountered AvroSerdeException determining schema. Returning " + "signal schema to indicate problem: " + he.getMessage()); return schema = SchemaResolutionProblem.SIGNAL_BAD_SCHEMA; } catch (Exception e) { LOG.warn("Encountered exception determining schema. Returning signal " + "schema to indicate problem", e); configErrors = new String("Encountered exception determining schema. Returning signal " + "schema to indicate problem: " + e.getMessage()); return SchemaResolutionProblem.SIGNAL_BAD_SCHEMA; } }
/** * Attempt to determine the schema via the usual means, but do not throw * an exception if we fail. Instead, signal failure via a special * schema. This is used because Hive calls init on the serde during * any call, including calls to update the serde properties, meaning * if the serde is in a bad state, there is no way to update that state. */ public Schema determineSchemaOrReturnErrorSchema(Configuration conf, Properties props) { try { configErrors = ""; return AvroSerdeUtils.determineSchemaOrThrowException(conf, props); } catch(AvroSerdeException he) { LOG.warn("Encountered AvroSerdeException determining schema. Returning " + "signal schema to indicate problem", he); configErrors = new String("Encountered AvroSerdeException determining schema. Returning " + "signal schema to indicate problem: " + he.getMessage()); return schema = SchemaResolutionProblem.SIGNAL_BAD_SCHEMA; } catch (Exception e) { LOG.warn("Encountered exception determining schema. Returning signal " + "schema to indicate problem", e); configErrors = new String("Encountered exception determining schema. Returning signal " + "schema to indicate problem: " + e.getMessage()); return SchemaResolutionProblem.SIGNAL_BAD_SCHEMA; } }
/** * Attempt to determine the schema via the usual means, but do not throw * an exception if we fail. Instead, signal failure via a special * schema. This is used because Hive calls init on the serde during * any call, including calls to update the serde properties, meaning * if the serde is in a bad state, there is no way to update that state. */ public Schema determineSchemaOrReturnErrorSchema(Configuration conf, Properties props) { try { configErrors = ""; return AvroSerdeUtils.determineSchemaOrThrowException(conf, props); } catch(AvroSerdeException he) { LOG.warn("Encountered AvroSerdeException determining schema. Returning " + "signal schema to indicate problem", he); configErrors = new String("Encountered AvroSerdeException determining schema. Returning " + "signal schema to indicate problem: " + he.getMessage()); return schema = SchemaResolutionProblem.SIGNAL_BAD_SCHEMA; } catch (Exception e) { LOG.warn("Encountered exception determining schema. Returning signal " + "schema to indicate problem", e); configErrors = new String("Encountered exception determining schema. Returning signal " + "schema to indicate problem: " + e.getMessage()); return SchemaResolutionProblem.SIGNAL_BAD_SCHEMA; } }
@Override public org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter getHiveRecordWriter(JobConf jobConf, Path path, Class<? extends Writable> valueClass, boolean isCompressed, Properties properties, Progressable progressable) throws IOException { Schema schema; try { schema = AvroSerdeUtils.determineSchemaOrThrowException(jobConf, properties); } catch (AvroSerdeException e) { throw new IOException(e); } GenericDatumWriter<GenericRecord> gdw = new GenericDatumWriter<GenericRecord>(schema); DataFileWriter<GenericRecord> dfw = new DataFileWriter<GenericRecord>(gdw); if (isCompressed) { int level = jobConf.getInt(DEFLATE_LEVEL_KEY, DEFAULT_DEFLATE_LEVEL); String codecName = jobConf.get(OUTPUT_CODEC, DEFLATE_CODEC); CodecFactory factory = codecName.equals(DEFLATE_CODEC) ? CodecFactory.deflateCodec(level) : CodecFactory.fromString(codecName); dfw.setCodec(factory); } dfw.create(schema, path.getFileSystem(jobConf).create(path)); return new AvroGenericRecordWriter(dfw); }
@Override public FileSinkOperator.RecordWriter getHiveRecordWriter(JobConf jobConf, Path path, Class<? extends Writable> valueClass, boolean isCompressed, Properties properties, Progressable progressable) throws IOException { Schema schema; try { schema = AvroSerdeUtils.determineSchemaOrThrowException(properties); } catch (AvroSerdeException e) { throw new IOException(e); } GenericDatumWriter<GenericRecord> gdw = new GenericDatumWriter<GenericRecord>(schema); DataFileWriter<GenericRecord> dfw = new DataFileWriter<GenericRecord>(gdw); if (isCompressed) { int level = jobConf.getInt(DEFLATE_LEVEL_KEY, DEFAULT_DEFLATE_LEVEL); String codecName = jobConf.get(OUTPUT_CODEC, DEFLATE_CODEC); CodecFactory factory = codecName.equals(DEFLATE_CODEC) ? CodecFactory.deflateCodec(level) : CodecFactory.fromString(codecName); dfw.setCodec(factory); } dfw.create(schema, path.getFileSystem(jobConf).create(path)); return new AvroGenericRecordWriter(dfw); }
return AvroSerdeUtils.determineSchemaOrThrowException(props); } else return null; // If it's not in this property, it won't be in any others
return AvroSerdeUtils.determineSchemaOrThrowException(job, props);