byte[] rawData = bufferOutStream.toByteArray(); IOUtils.closeQuietly(bufferOutStream); FetchedData fetchedData = new FetchedData(rawData, urlConn.getContentType(), responseCode); resource.setStatus(ResourceStatus.FETCHED.toString()); fetchedData.setResource(resource); fetchedData.setHeaders(urlConn.getHeaderFields()); if (truncated) { fetchedData.getHeaders().put(TRUNCATED, Collections.singletonList(Boolean.TRUE.toString()));
@Override public FetchedData apply(Resource resource) { try { return this.fetch(resource); } catch (Exception e) { int statusCode = DEFAULT_ERROR_CODE; if (e instanceof FileNotFoundException){ statusCode = 404; } LOG.warn("FETCH-ERROR {}", resource.getUrl()); LOG.debug(e.getMessage(), e); FetchedData fetchedData = new FetchedData(new byte[0], "", statusCode); resource.setStatus(ResourceStatus.ERROR.toString()); fetchedData.setResource(resource); return fetchedData; } } }
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { IOUtils.copy(boundedStream, out); fetchedData = new FetchedData(out.toByteArray(), response.getContentType(), response.getStatusCode()); long contentLength = page.getWebResponse().getContentLength(); if (contentLength > 0 && contentLength < Integer.MAX_VALUE) { fetchedData.setContentLength((int) contentLength); truncated = (contentLength > fetchedData.getContentLength()); if (truncated) { LOG.info("Content Truncated: {}, TotalSize={}", resource.getUrl(), contentLength); fetchedData.setHeaders(headers); if (respHeaders != null && !respHeaders.isEmpty()){ respHeaders.forEach(item -> { fetchedData = new FetchedData(new byte[0], "unknown/unknown", 0); // fixme: use proper status code resource.setStatus(ResourceStatus.ERROR.toString()); fetchedData.setResource(resource); return fetchedData;
return super.fetch(resource); fetchedData = new FetchedData(html.getBytes(), "application/html", status); resource.setStatus(ResourceStatus.FETCHED.toString()); fetchedData.setResource(resource); return fetchedData;