/** * Set number of schema evolution DDLs as Sla event metadata */ public static void setEvolutionMetadata(State state, List<String> evolutionDDLs) { state.setProp(EventConstants.SCHEMA_EVOLUTION_DDLS_NUM, evolutionDDLs == null ? 0 : evolutionDDLs.size()); }
/** * Disable Hive watermarker. This is necessary when there is no concrete source table where watermark can be inferred. */ public static void disableHiveWatermarker(State state) { state.setProp(USE_WATERMARKER_KEY, Boolean.toString(false)); }
/*** * Serialize a {@link QueryBasedHivePublishEntity} into a {@link State} at {@link #SERIALIZED_PUBLISH_TABLE_COMMANDS}. * @param state {@link State} to serialize entity into. * @param queryBasedHivePublishEntity to carry to publisher. */ public static void serializePublishCommands(State state, QueryBasedHivePublishEntity queryBasedHivePublishEntity) { state.setProp(HiveAvroORCQueryGenerator.SERIALIZED_PUBLISH_TABLE_COMMANDS, GSON.toJson(queryBasedHivePublishEntity)); }
/** * Given a {@link String} in octal notation, set a key, value pair in the given {@link State} for the writer to * use when creating files. This method should be used in conjunction with {@link #deserializeWriterFilePermissions(State, int, int)}. */ public static void setWriterFileOctalPermissions(State state, int numBranches, int branchId, String octalPermissions) { state.setProp( ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_FILE_PERMISSIONS, numBranches, branchId), octalPermissions); }
public static State getStateFromConf(Configuration conf) { State state = new State(); for (Entry<String, String> entry : conf) { state.setProp(entry.getKey(), entry.getValue()); } return state; }
/** * Add the input file to the Hive session before running the task. */ public static void addFile(State state, String file) { state.setProp(ADD_FILES, state.getProp(ADD_FILES, "") + "," + file); }
/** * Add the input jar to the Hive session before running the task. */ public static void addJar(State state, String jar) { state.setProp(ADD_JARS, state.getProp(ADD_JARS, "") + "," + jar); }
@Override public State getFinalState() { State state = new State(); state.setProp("RecordsWritten", recordsWritten()); try { state.setProp("BytesWritten", bytesWritten()); } catch (Exception exception) { // If Writer fails to return bytesWritten, it might not be implemented, or implemented incorrectly. // Omit property instead of failing. } return state; }
public static FileSystem getOwnerFs(State state, Optional<String> owner) throws IOException { if (owner.isPresent()) { state.setProp(ComplianceConfigurationKeys.GOBBLIN_COMPLIANCE_PROXY_USER, owner.get()); } ProxyUtils.setProxySettingsForFs(state); return WriterUtils.getWriterFs(state); }
/** * Write record count to a specific directory. * File name is {@link InputRecordCountHelper#RECORD_COUNT_FILE} * @param fs file system in use * @param dir directory where a record file is located */ @Deprecated public static void writeRecordCount (FileSystem fs, Path dir, long count) throws IOException { State state = loadState(fs, dir); state.setProp(CompactionSlaEventHelper.RECORD_COUNT_TOTAL, count); saveState(fs, dir, state); }
private State getTestState(long recordsRead, long recordsWritten, double range) { State state = new State(); state.setProp(ConfigurationKeys.EXTRACTOR_ROWS_EXPECTED, recordsRead); state.setProp(ConfigurationKeys.WRITER_ROWS_WRITTEN, recordsWritten); state.setProp(ConfigurationKeys.ROW_COUNT_RANGE, range); return state; } }
@Override public State getFinalState() { State state = new State(); if (this.writer instanceof FinalState) { state.addAll(((FinalState)this.writer).getFinalState()); } else { LOG.warn("Wrapped writer does not implement FinalState: " + this.writer.getClass()); } state.setProp(THROTTLED_TIME_KEY, this.throttledTime); return state; }
@Override public State getFinalState() { State state = new State(); if (this.writer instanceof FinalState) { state.addAll(((FinalState)this.writer).getFinalState()); } else { LOG.warn("Wrapped writer does not implement FinalState: " + this.writer.getClass()); } state.setProp(FAILED_WRITES_KEY, this.failedWrites); return state; }
/** * Create a CheckpointableWatermarkState object from a CheckpointableWatermark * @param watermark: the checkpointable watermark * @param gson: the instance of {@link Gson} to use for serializing the {@param watermark}. */ public CheckpointableWatermarkState(CheckpointableWatermark watermark, Gson gson) { super.setProp(watermark.getSource(), gson.toJsonTree(watermark)); super.setId(watermark.getSource()); }
/** * Set the average time in milliseconds to pull a record of a partition, which will be stored in property * "[topicname].[partitionid].avg.record.millis". */ public static void setPartitionAvgRecordMillis(State state, KafkaPartition partition, double millis) { state.setProp( getPartitionPropName(partition.getTopicName(), partition.getId()) + "." + KafkaSource.AVG_RECORD_MILLIS, millis); }
@Test public void testPolicyChecker() throws Exception { State state = new State(); state.setProp(ConfigurationKeys.TASK_LEVEL_POLICY_LIST, "org.apache.gobblin.qualitychecker.TestTaskLevelPolicy"); state.setProp(ConfigurationKeys.TASK_LEVEL_POLICY_LIST_TYPE, "FAIL"); TaskLevelPolicyCheckResults results = getPolicyResults(state); for (Map.Entry<TaskLevelPolicy.Result, TaskLevelPolicy.Type> entry : results.getPolicyResults().entrySet()) { Assert.assertEquals(entry.getKey(), TaskLevelPolicy.Result.PASSED); } }
@Test public void testMultiplePolicies() throws Exception { State state = new State(); state.setProp(ConfigurationKeys.TASK_LEVEL_POLICY_LIST, "org.apache.gobblin.qualitychecker.TestTaskLevelPolicy,org.apache.gobblin.qualitychecker.TestTaskLevelPolicy"); state.setProp(ConfigurationKeys.TASK_LEVEL_POLICY_LIST_TYPE, "FAIL,FAIL"); TaskLevelPolicyCheckResults results = getPolicyResults(state); for (Map.Entry<TaskLevelPolicy.Result, TaskLevelPolicy.Type> entry : results.getPolicyResults().entrySet()) { Assert.assertEquals(entry.getKey(), TaskLevelPolicy.Result.PASSED); } }
@Test public void testGetPersistDir() throws Exception { State state = new State(); Assert.assertFalse(RecoveryHelper.getPersistDir(state).isPresent()); state.setProp(RecoveryHelper.PERSIST_DIR_KEY, this.tmpDir.getAbsolutePath()); Assert.assertTrue(RecoveryHelper.getPersistDir(state).isPresent()); Assert.assertTrue(RecoveryHelper.getPersistDir(state).get().toUri().getPath(). startsWith(this.tmpDir.getAbsolutePath())); }
public void userCreatedStagingTable() throws SQLException { this.state.setProp(ConfigurationKeys.WRITER_STAGING_TABLE, STAGING_TABLE); when(this.commands.isEmpty(DB, STAGING_TABLE)).thenReturn(Boolean.TRUE); this.initializer.initialize(); Assert.assertEquals(STAGING_TABLE, this.workUnit.getProp(ConfigurationKeys.WRITER_STAGING_TABLE)); verify(this.commands, never()).createTableStructure(anyString(), anyString(), anyString()); verify(this.commands, never()).truncate(anyString(), anyString()); verify(this.commands, never()).drop(anyString(), anyString()); }
@Test public void testConfigureFromState() { HttpClientConfiguratorLoader loader = new HttpClientConfiguratorLoader(new State()); Assert.assertEquals(loader.getConfigurator().getClass(), DefaultHttpClientConfigurator.class); State state = new State(); state.setProp(HttpClientConfiguratorLoader.HTTP_CLIENT_CONFIGURATOR_TYPE_FULL_KEY, "default"); loader = new HttpClientConfiguratorLoader(state); Assert.assertEquals(loader.getConfigurator().getClass(), DefaultHttpClientConfigurator.class); }