/** * Generate ZipInputStream with InputStream * * @param stream source file InputStream * @return a new ZIP input stream * @throws IOException */ public static ZipInputStream getZipInputStream(InputStream stream) throws IOException { return createZipInputStream(stream); }
@Test public void testCompareFile() throws Exception { String resources = getClass().getResource("/compare").toURI().getPath(); LOG.info("Resources folder: " + resources); // Text content and Binary both same String srcFile = resources + "/source.csv"; String refFile = resources + "/bin_same_ref.csv"; Assert.assertTrue(FileRuntimeHelper.compareInTextMode(srcFile, refFile, "UTF-8")); Assert.assertTrue(FileRuntimeHelper.compareInBinaryMode(srcFile, refFile)); // Text content are same and Binary is different srcFile = resources + "/prenoms_fr2.csv.short.dos"; refFile = resources + "/prenoms_fr2.csv.short.linux"; Assert.assertTrue(FileRuntimeHelper.compareInTextMode(srcFile, refFile, "UTF-8")); Assert.assertFalse(FileRuntimeHelper.compareInBinaryMode(srcFile, refFile)); // Text are different srcFile = resources + "/source.csv"; refFile = resources + "/text_diff_ref.csv"; Assert.assertFalse(FileRuntimeHelper.compareInTextMode(srcFile, refFile, "UTF-8")); Assert.assertFalse(FileRuntimeHelper.compareInBinaryMode(srcFile, refFile)); }
@Test public void testFileUncompress() throws Throwable { String resources = getClass().getResource("/zip").toURI().getPath(); LOG.info("Resources folder: " + resources); String zipFile = resources + "/test_uncompress.zip"; ZipInputStream zipInputStream = FileRuntimeHelper.getZipInputStream(zipFile); int entryCount = 0; List<String> entries = Arrays.asList("folder_1/file_1_1.txt", "folder_1/file_1_2.csv", "folder_2/file_2_1.txt", "folder_2/file_2_2.csv", "test.txt"); while (true) { ZipEntry entry = FileRuntimeHelper.getCurrentZipEntry(zipInputStream); if (entry == null) { break; } entryCount++; LOG.debug("Entry name: " + entry.getName()); Assert.assertTrue(entries.contains(entry.getName())); } LOG.debug("Entries count: " + entryCount); Assert.assertEquals(5, entryCount); } }
protected void basicOutputTest(TFileOutputDelimitedProperties properties, String refFilePath) throws Throwable { String fileName = properties.fileName.getStringValue(); if (properties.targetIsStream.getValue()) { properties.fileName.setValue(new FileOutputStream(new File(fileName))); } List<IndexedRecord> records = generateRecords(25); Result result = doWriteRows(properties, records); assertEquals(25, result.getTotalCount()); String outputFile = fileName; if (properties.compress.getValue()) { outputFile = outputFile.substring(0, outputFile.lastIndexOf(".")) + ".zip"; } assertTrue(FileRuntimeHelper.compareInTextMode(outputFile, refFilePath, getEncoding(properties.encoding))); // Need to close the stream firstly. then if (properties.targetIsStream.getValue()) { ((FileOutputStream) properties.fileName.getValue()).close(); } assertTrue(deleteFile(fileName)); }
public void init() throws IOException { fileNameOrStream = props.fileName.getValue(); sourceIsStream = fileNameOrStream instanceof InputStream; encoding = getEncoding(); schemaIsDynamic = AvroUtils.isIncludeAllFields(props.main.schema.getValue()); header = (props.header.getValue() == null) ? -1 : props.header.getValue(); if (schemaIsDynamic) { header = header - 1; } if (props instanceof TFileInputDelimitedProperties) { uncompress = ((TFileInputDelimitedProperties) props).uncompress.getValue(); splitRecord = ((TFileInputDelimitedProperties) props).splitRecord.getValue(); if (((TFileInputDelimitedProperties) props).random.getValue()) { nbRandom = (((TFileInputDelimitedProperties) props).nbRandom.getValue() == null || uncompress) ? -1 : ((TFileInputDelimitedProperties) props).nbRandom.getValue(); } } footer = (props.footer.getValue() == null || uncompress) ? -1 : props.footer.getValue(); limit = (props.limit.getValue() == null) ? -1 : props.limit.getValue(); if (sourceIsStream) { zipInputStream = FileRuntimeHelper.getZipInputStream((InputStream) fileNameOrStream); } else { zipInputStream = FileRuntimeHelper.getZipInputStream(String.valueOf(fileNameOrStream)); } fieldSeparator = getFieldSeparator(); rowSeparator = getRowSeparator(); setEscapeAndTextEnclosure(props.escapeChar.getValue(), props.textEnclosure.getValue()); }
@Test public void testWriteDecimal() throws Throwable { String resources = getResourceFolder(); String outputFile = resources + "/out/test_write_decimal.csv"; String refFilePath = resources + "/ref_test_write_decimal.csv"; LOGGER.debug("Test file path: " + outputFile); TFileOutputDelimitedProperties properties = createOutputProperties(outputFile, false); Schema outputSchema = SchemaBuilder.builder().record("Schema").fields().name("TestBigDecimal") .prop(SchemaConstants.TALEND_COLUMN_PRECISION, "10").prop(SchemaConstants.TALEND_COLUMN_PRECISION, "2") .type(AvroUtils._decimal()).noDefault().endRecord(); properties.main.schema.setValue(outputSchema); List<IndexedRecord> records = new ArrayList<>(); IndexedRecord r1 = new GenericData.Record(outputSchema); r1.put(0, "3.1415926"); IndexedRecord r2 = new GenericData.Record(outputSchema); r2.put(0, "9.1798"); records.add(r1); records.add(r2); // Delete generated empty file function not be checked doWriteRows(properties, records); assertTrue(FileRuntimeHelper.compareInTextMode(outputFile, refFilePath, getEncoding(properties.encoding))); assertTrue(deleteFile(outputFile)); }
/** * Generate ZipInputStream with file * * @param fileName zip file name * @return a new ZIP input stream * @throws IOException */ public static ZipInputStream getZipInputStream(String fileName) throws IOException { return createZipInputStream(fileName); }
/** * Generate ZipInputStream with file * * @param fileName zip file name * @return a new ZIP input stream * @throws IOException */ public static ZipInputStream getZipInputStream(String fileName) throws IOException { return createZipInputStream(fileName); }
/** * Generate ZipInputStream with InputStream * * @param stream source file InputStream * @return a new ZIP input stream * @throws IOException */ public static ZipInputStream getZipInputStream(InputStream stream) throws IOException { return createZipInputStream(stream); }