@Override public RequiredFieldResponse pushProjection(RequiredFieldList requiredFieldList) throws FrontendException { String fields = PigUtils.asProjection(requiredFieldList, properties); getUDFProperties().setProperty(InternalConfigurationOptions.INTERNAL_ES_TARGET_FIELDS, fields); if (log.isTraceEnabled()) { log.trace(String.format("Given push projection; saving field projection [%s]", fields)); } return new RequiredFieldResponse(true); }
@Override public RequiredFieldResponse pushProjection(RequiredFieldList requiredFieldList) { LOG.info(format("[%s]: pushProjection() -> %s", signature, requiredFieldList)); try { List<String> projection = requiredFieldList.getFields().stream().map(RequiredField::getAlias).collect(Collectors.toList()); storeInUDFContext(ICEBERG_PROJECTED_FIELDS, (Serializable) projection); } catch (IOException e) { throw new RuntimeException(e); } return new RequiredFieldResponse(true); }
@Override public RequiredFieldResponse pushProjection(RequiredFieldList requiredFieldList) throws FrontendException { String fields = PigUtils.asProjection(requiredFieldList, properties); getUDFProperties().setProperty(InternalConfigurationOptions.INTERNAL_ES_TARGET_FIELDS, fields); if (log.isTraceEnabled()) { log.trace(String.format("Given push projection; saving field projection [%s]", fields)); } return new RequiredFieldResponse(true); }
return new RequiredFieldResponse(true); return new RequiredFieldResponse(true);
@Override public RequiredFieldResponse pushProjection(RequiredFieldList requiredFieldList) throws FrontendException { if (requiredFieldList == null) return null; if (requiredFieldList.getFields() != null) { int lastColumn = -1; for (RequiredField rf: requiredFieldList.getFields()) { if (rf.getIndex()>lastColumn) { lastColumn = rf.getIndex(); } } mRequiredColumns = new boolean[lastColumn+1]; for (RequiredField rf: requiredFieldList.getFields()) { if (rf.getIndex()!=-1) mRequiredColumns[rf.getIndex()] = true; } Properties p = UDFContext.getUDFContext().getUDFProperties(this.getClass()); try { p.setProperty(signature, ObjectSerializer.serialize(mRequiredColumns)); } catch (Exception e) { throw new RuntimeException("Cannot serialize mRequiredColumns"); } } return new RequiredFieldResponse(true); }
@Override public RequiredFieldResponse pushProjection( RequiredFieldList requiredFieldList) throws FrontendException { if (requiredFieldList == null) return null; if (requiredFieldList.getFields() != null) { int schemaSize = ((StructTypeInfo)typeInfo).getAllStructFieldTypeInfos().size(); mRequiredColumns = new boolean[schemaSize]; for (RequiredField rf: requiredFieldList.getFields()) { if (rf.getIndex()!=-1) mRequiredColumns[rf.getIndex()] = true; } Properties p = UDFContext.getUDFContext().getUDFProperties(this.getClass()); try { p.setProperty(signature + RequiredColumnsSuffix, ObjectSerializer.serialize(mRequiredColumns)); } catch (Exception e) { throw new RuntimeException("Cannot serialize mRequiredColumns"); } } return new RequiredFieldResponse(true); }
/** * A helper method for implementing * {@link LoadPushDown#pushProjection(RequiredFieldList)}. <p> * * Stores requiredFieldList in context. The requiredFields are read from * context on the backend (in side {@link #setLocation(String, Job)}). */ protected RequiredFieldResponse pushProjectionHelper( RequiredFieldList requiredFieldList) throws FrontendException { try { getUDFProperties().setProperty(projectionKey, ObjectSerializer.serialize(requiredFieldList)); } catch (IOException e) { // not expected throw new FrontendException(e); } return new RequiredFieldResponse(true); }
@Override public RequiredFieldResponse pushProjection(final RequiredFieldList rfl) throws FrontendException { requiredFieldList = rfl; Schema newSchema = AvroStorageSchemaConversionUtilities .newSchemaFromRequiredFieldList(schema, rfl); if (newSchema != null) { schema = newSchema; setInputAvroSchema(schema); return new RequiredFieldResponse(true); } else { log.warn("could not select fields subset " + rfl + "\n"); warn("could not select fields subset", PigWarning.UDF_WARNING_2); return new RequiredFieldResponse(false); } }
@Override public RequiredFieldResponse pushProjection(RequiredFieldList requiredFieldsInfo) throws FrontendException { // Store the required fields information in the UDFContext so that we // can retrieve it later. storeInUDFContext(signature, PRUNE_PROJECTION_INFO, requiredFieldsInfo); // HCat will always prune columns based on what we ask of it - so the // response is true return new RequiredFieldResponse(true); }
@Override public RequiredFieldResponse pushProjection(RequiredFieldList pRequiredFieldList) { this.requiredFieldList = pRequiredFieldList; // Store the required fields information in the UDFContext. storeInUDFContext(PRUNE_PROJECTION_INFO, this.requiredFieldList); return new RequiredFieldResponse(true); }
@Override public RequiredFieldResponse pushProjection(RequiredFieldList requiredFieldList) throws FrontendException { this.requiredFieldList = requiredFieldList; if (requiredFieldList == null) return null; schema = getSchemaFromRequiredFieldList(schema, requiredFieldList.getFields()); storeInUDFContext(PARQUET_PIG_SCHEMA, pigSchemaToString(schema)); storeInUDFContext(PARQUET_PIG_REQUIRED_FIELDS, serializeRequiredFieldList(requiredFieldList)); return new RequiredFieldResponse(true); }
@Override public RequiredFieldResponse pushProjection(RequiredFieldList requiredFieldsInfo) throws FrontendException { // Store the required fields information in the UDFContext so that we // can retrieve it later. storeInUDFContext(signature, PRUNE_PROJECTION_INFO, requiredFieldsInfo); // HCat will always prune columns based on what we ask of it - so the // response is true return new RequiredFieldResponse(true); }
@Override public RequiredFieldResponse pushProjection(RequiredFieldList pRequiredFieldList) { this.requiredFieldList = pRequiredFieldList; // Store the required fields information in the UDFContext. storeInUDFContext(PRUNE_PROJECTION_INFO, this.requiredFieldList); return new RequiredFieldResponse(true); }
@Override public RequiredFieldResponse pushProjection(RequiredFieldList requiredFieldsInfo) throws FrontendException { // Store the required fields information in the UDFContext so that we // can retrieve it later. storeInUDFContext(signature, PRUNE_PROJECTION_INFO, requiredFieldsInfo); // HCat will always prune columns based on what we ask of it - so the // response is true return new RequiredFieldResponse(true); }