public static void writeTmpTable(TajoConf conf, Path tablePath) throws IOException { FileTablespace sm = TablespaceManager.getDefault(); Appender appender; Path filePath = new Path(tablePath, "table.csv"); FileSystem fs = filePath.getFileSystem(conf); if (fs.exists(tablePath)) { fs.delete(tablePath, true); } fs.mkdirs(tablePath); appender = sm.getAppender(mockupMeta, mockupSchema, filePath); appender.init(); int deptSize = 10000; int tupleNum = 100; Tuple tuple; for (int i = 0; i < tupleNum; i++) { tuple = new VTuple(2); String key = "test" + (i % deptSize); tuple.put(0, DatumFactory.createText(key)); tuple.put(1, DatumFactory.createInt4(i + 1)); appender.addTuple(tuple); } appender.close(); } }
public void openAppender(int suffixId) throws IOException { Path actualFilePath = lastFileName; if (suffixId > 0) { actualFilePath = new Path(lastFileName + "_" + suffixId); } appender = ((FileTablespace) TablespaceManager.get(lastFileName.toUri())) .getAppender(meta, outSchema, actualFilePath); appender.enableStats(); appender.init(); }
@Override public Tuple next() throws IOException { Tuple tuple; while(!context.isStopped() && (tuple = child.next()) != null) { appender.addTuple(tuple); if (maxPerFileSize > 0 && maxPerFileSize <= appender.getEstimatedOutputSize()) { appender.close(); writtenFileNum++; StatisticsUtil.aggregateTableStat(sumStats, appender.getStats()); openNewFile(writtenFileNum); } } return null; }
@Override public Tuple next() throws IOException { Tuple tuple; while(!context.isStopped() && (tuple = child.next()) != null) { partKey.set(tuple); // add tuple getAppender(partKey, tuple).addTuple(tuple); } List<TableStats> statSet = new ArrayList<>(); for (Appender app : appenderMap.values()) { app.flush(); app.close(); statSet.add(app.getStats()); } // Collect and aggregated statistics data TableStats aggregated = StatisticsUtil.aggregateTableStat(statSet); context.setResultStats(aggregated); return null; } }
@Test public void testFileAlreadyExists() throws IOException { if (internalType) return; Schema schema = SchemaBuilder.builder() .add("id", Type.INT4) .add("age", Type.INT8) .add("score", Type.FLOAT4) .build(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf); if (dataFormat.equalsIgnoreCase(BuiltinStorages.AVRO)) { meta.putProperty(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_PROJECTION_AVRO_SCHEMA); } FileTablespace sm = TablespaceManager.getLocalFs(); Path tablePath = new Path(testDir, "testFileAlreadyExists.data"); Appender appender = sm.getAppender(meta, schema, tablePath); appender.init(); appender.close(); try { appender = sm.getAppender(meta, schema, tablePath); appender.init(); if (BuiltinStorages.ORC.equals(dataFormat)) { appender.close(); } fail(dataFormat); } catch (IOException e) { } finally { IOUtils.cleanup(null, appender); } }
@Override public Tuple next() throws IOException { while((tuple = child.next()) != null) { appender.addTuple(tuple); } return null; }
FileTablespace sm = TablespaceManager.getLocalFs(); Appender appender = sm.getAppender(meta, dataSchema, tablePath); appender.init(); }); appender.addTuple(expect); appender.flush(); appender.addTuple(expect2); appender.flush(); appender.close(); } catch (ValueTooLongForTypeCharactersException e) { ok = true;
@Override public Tuple next() throws IOException { Tuple tuple; while(!context.isStopped() && (tuple = child.next()) != null) { appender.addTuple(tuple); if (maxPerFileSize > 0 && maxPerFileSize <= appender.getEstimatedOutputSize()) { appender.close(); writtenFileNum++; StatisticsUtil.aggregateTableStat(sumStats, appender.getStats()); openNewFile(writtenFileNum); } } return null; }
@Override public Tuple next() throws IOException { Tuple tuple; while(!context.isStopped() && (tuple = child.next()) != null) { partKey.set(tuple); // add tuple getAppender(partKey, tuple).addTuple(tuple); } List<TableStats> statSet = new ArrayList<TableStats>(); for (Appender app : appenderMap.values()) { app.flush(); app.close(); statSet.add(app.getStats()); } // Collect and aggregated statistics data TableStats aggregated = StatisticsUtil.aggregateTableStat(statSet); context.setResultStats(aggregated); return null; } }
@Override public Tuple next() throws IOException { while((tuple = child.next()) != null) { appender.addTuple(tuple); } return null; }
Appender appender = ((FileTablespace) TablespaceManager.getLocalFs()) .getAppender(scoreLargeMeta, scoreSchmea, scoreLargePath); appender.enableStats(); appender.init(); largeScore = new TableDesc( IdentifierUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "score_large"), scoreSchmea, scoreLargeMeta, appender.addTuple(tuple); m++; appender.flush(); appender.close(); largeScore.setStats(appender.getStats()); catalog.createTable(largeScore);
@Override public Tuple next() throws IOException { Tuple tuple; while(!context.isStopped() && (tuple = child.next()) != null) { if (prevKey == null) { appender = getNextPartitionAppender(getSubdirectory(tuple)); prevKey = new ComparableTuple(inSchema, keyIds); prevKey.set(tuple); } else if (!prevKey.equals(tuple)) { appender.close(); StatisticsUtil.aggregateTableStat(aggregatedStats, appender.getStats()); appender = getNextPartitionAppender(getSubdirectory(tuple)); prevKey.set(tuple); // reset all states for file rotating writtenFileNum = 0; } appender.addTuple(tuple); if (maxPerFileSize > 0 && maxPerFileSize <= appender.getEstimatedOutputSize()) { appender.close(); writtenFileNum++; StatisticsUtil.aggregateTableStat(aggregatedStats, appender.getStats()); openAppender(writtenFileNum); } } return null; }
public void openAppender(int suffixId) throws IOException { Path actualFilePath = lastFileName; if (suffixId > 0) { actualFilePath = new Path(lastFileName + "_" + suffixId); } appender = ((FileTablespace) TablespaceManager.get(lastFileName.toUri())) .getAppender(meta, outSchema, actualFilePath); appender.enableStats(); appender.init(); }
FileTablespace sm = TablespaceManager.getLocalFs(); Appender appender = sm.getAppender(meta, schema, tablePath); appender.enableStats(); appender.init(); appender.close(); TableStats stat = appender.getStats(); assertEquals(0, stat.getNumRows().longValue());