protected boolean sql(CsvData data) { try { statistics.get(batch).startTimer(DataWriterStatisticConstants.DATABASEMILLIS); String script = data.getParsedData(CsvData.ROW_DATA)[0]; List<String> sqlStatements = getSqlStatements(script); long count = 0; for (String sql : sqlStatements) { sql = preprocessSqlStatement(sql); transaction.prepare(sql); if (log.isDebugEnabled()) { log.debug("About to run: {}", sql); } count += transaction.prepareAndExecute(sql); if (log.isDebugEnabled()) { log.debug("{} rows updated when running: {}", count, sql); } } statistics.get(batch).increment(DataWriterStatisticConstants.SQLCOUNT); statistics.get(batch).increment(DataWriterStatisticConstants.SQLROWSAFFECTEDCOUNT, count); return true; } finally { statistics.get(batch).stopTimer(DataWriterStatisticConstants.DATABASEMILLIS); } }
public void start(Batch batch) { this.batch = batch; this.statistics.put(batch, new Statistics()); }
public void setValues(Statistics readerStatistics, Statistics writerStatistics, boolean isSuccess) { if (readerStatistics != null) { byteCount = readerStatistics.get(DataReaderStatistics.READ_BYTE_COUNT); } if (writerStatistics != null) { filterMillis = writerStatistics.get(DataWriterStatisticConstants.FILTERMILLIS); databaseMillis = writerStatistics.get(DataWriterStatisticConstants.DATABASEMILLIS); statementCount = writerStatistics.get(DataWriterStatisticConstants.STATEMENTCOUNT); fallbackInsertCount = writerStatistics .get(DataWriterStatisticConstants.FALLBACKINSERTCOUNT); fallbackUpdateCount = writerStatistics .get(DataWriterStatisticConstants.FALLBACKUPDATECOUNT); missingDeleteCount = writerStatistics .get(DataWriterStatisticConstants.MISSINGDELETECOUNT); ignoreCount = writerStatistics.get(DataWriterStatisticConstants.IGNORECOUNT); lastUpdatedTime = new Date(); if (!isSuccess) { failedRowNumber = statementCount; failedLineNumber = writerStatistics.get(DataWriterStatisticConstants.LINENUMBER); } } }
protected void notifyFiltersEarlyCommit() { List<IDatabaseWriterFilter> filters = this.writerSettings.getDatabaseWriterFilters(); if (filters != null) { try { statistics.get(batch).startTimer(DataWriterStatisticConstants.FILTERMILLIS); for (IDatabaseWriterFilter filter : filters) { filter.earlyCommit(context); } } finally { statistics.get(batch).stopTimer(DataWriterStatisticConstants.FILTERMILLIS); } } }
public void increment(String category) { increment(category, 1); }
protected boolean script(CsvData data) { try { statistics.get(batch).startTimer(DataWriterStatisticConstants.DATABASEMILLIS); String script = data.getParsedData(CsvData.ROW_DATA)[0]; Map<String, Object> variables = new HashMap<String, Object>(); statistics.get(batch).increment(DataWriterStatisticConstants.SCRIPTCOUNT); } catch (EvalError e) { throw new RuntimeException(e);
protected void flush() { statistics.get(batch).startTimer(DataWriterStatisticConstants.DATABASEMILLIS); try { if (rowArrays.size() > 0) { statistics.get(batch).set(DataWriterStatisticConstants.STATEMENTCOUNT, errors[0]); lastEventType = null; rowArrays.clear(); statistics.get(batch).stopTimer(DataWriterStatisticConstants.DATABASEMILLIS);
Conflict conflict = writerSettings.pickConflict(writer.getTargetTable(), writer.getBatch()); Statistics statistics = writer.getStatistics().get(writer.getBatch()); long statementCount = statistics.get(DataWriterStatisticConstants.STATEMENTCOUNT); long lineNumber = statistics.get(DataWriterStatisticConstants.LINENUMBER); ResolvedData resolvedData = writerSettings.getResolvedData(statementCount); .increment(DataWriterStatisticConstants.MISSINGDELETECOUNT);
protected void notifyFiltersBatchRolledback() { List<IDatabaseWriterFilter> filters = this.writerSettings.getDatabaseWriterFilters(); if (filters != null) { try { statistics.get(batch).startTimer(DataWriterStatisticConstants.FILTERMILLIS); for (IDatabaseWriterFilter filter : filters) { filter.batchRolledback(context); } } finally { statistics.get(batch).stopTimer(DataWriterStatisticConstants.FILTERMILLIS); } } }
public long stopTimer(String category) { long time = 0; Long startTime = timers.get(category); if (startTime != null) { time = System.currentTimeMillis() - startTime; increment(category, time); } timers.remove(category); return time; }
case INSERT: startCopy(); statistics.get(batch).increment(DataWriterStatisticConstants.STATEMENTCOUNT); statistics.get(batch).increment(DataWriterStatisticConstants.LINENUMBER); statistics.get(batch).startTimer(DataWriterStatisticConstants.DATABASEMILLIS); try { String formattedData = CsvUtils.escapeCsvData( throw getPlatform().getSqlTemplate().translate(ex); } finally { statistics.get(batch).stopTimer(DataWriterStatisticConstants.DATABASEMILLIS);
protected void rollback() { if (transaction != null) { try { statistics.get(batch).startTimer(DataWriterStatisticConstants.DATABASEMILLIS); this.transaction.rollback(); notifyFiltersBatchRolledback(); } finally { statistics.get(batch).stopTimer(DataWriterStatisticConstants.DATABASEMILLIS); } } uncommittedCount = 0; }
protected void println(String... data) { FileInfo fileInfo = fileInfoByTable.get(table.getFullyQualifiedTableName()); if (fileInfo != null) { try { StringBuilder buffer = new StringBuilder(); for (int i = 0; i < data.length; i++) { if (i != 0) { buffer.append(","); } buffer.append(data[i]); } buffer.append("\n"); fileInfo.outputFileWriter.write(buffer.toString()); long byteCount = buffer.length(); statistics.get(batch).increment(DataWriterStatisticConstants.BYTECOUNT, byteCount); } catch (IOException e) { throw new IoException(e); } } }
public void start(Batch batch) { this.batch = batch; this.statistics.put(batch, new Statistics()); }
public void end(Table table) { if (this.currentDataWriter != null) { this.currentDataWriter.end(table); Statistics stats = this.currentDataWriter.getStatistics().get(batch); this.outgoingBatch.setByteCount(stats.get(DataWriterStatisticConstants.BYTECOUNT)); } }
protected boolean create(CsvData data) { String xml = null; try { transaction.commit(); statistics.get(batch).startTimer(DataWriterStatisticConstants.DATABASEMILLIS); xml = data.getParsedData(CsvData.ROW_DATA)[0]; log.info("About to create table using the following definition: {}", xml); StringReader reader = new StringReader(xml); Database db = DatabaseXmlUtil.read(reader, false); if (writerSettings.isCreateTableAlterCaseToMatchDatabaseDefault()) { platform.alterCaseToMatchDatabaseDefaultCase(db); } if (writerSettings.isAlterTable()) { platform.alterDatabase(db, !writerSettings.isCreateTableFailOnError()); } else { platform.createDatabase(db, writerSettings.isCreateTableDropFirst(), !writerSettings.isCreateTableFailOnError()); } platform.resetCachedTableModel(); statistics.get(batch).increment(DataWriterStatisticConstants.CREATECOUNT); return true; } catch (RuntimeException ex) { log.error("Failed to alter table using the following xml: {}", xml); throw ex; } finally { statistics.get(batch).stopTimer(DataWriterStatisticConstants.DATABASEMILLIS); } }
protected void notifyFiltersBatchComplete() { List<IDatabaseWriterFilter> filters = this.writerSettings.getDatabaseWriterFilters(); if (filters != null) { try { statistics.get(batch).startTimer(DataWriterStatisticConstants.FILTERMILLIS); for (IDatabaseWriterFilter filter : filters) { filter.batchComplete(context); } } finally { statistics.get(batch).stopTimer(DataWriterStatisticConstants.FILTERMILLIS); } } }
protected long println(String... data) { long byteCount = 0; for (int i = 0; i < data.length; i++) { if (i != 0) { print(batch, delimiter); byteCount += delimiter.length(); } print(batch, data[i]); byteCount += data[i].length(); } print(batch, "\n"); byteCount += "\n".length(); statistics.get(batch).increment(DataWriterStatisticConstants.BYTECOUNT, byteCount); return byteCount; }
public void start(Batch batch) { this.batch = batch; this.statistics.put(batch, new Statistics()); this.snapshotEvents = new ArrayList<FileSnapshot>(); }
public void write(CsvData data) { this.outgoingBatch.incrementDataEventCount(); this.outgoingBatch.incrementInsertEventCount(); this.currentDataWriter.write(data); if (this.outgoingBatch.getDataEventCount() >= maxBatchSize && this.batches.size() > 0) { this.currentDataWriter.end(table); this.currentDataWriter.end(batch, false); Statistics stats = this.currentDataWriter.getStatistics().get(batch); this.outgoingBatch.setByteCount(stats.get(DataWriterStatisticConstants.BYTECOUNT)); this.currentDataWriter.close(); startNewBatch(); } }