@Override public void process(Status status) { URLEntity[] entities = status.getURLEntities(); if (entities != null) { for (URLEntity ue: entities) { if (ue != null) { // see why we intermittently get NPEs url.emit((ue.getExpandedURL() == null ? ue.getURL() : ue.getExpandedURL()).toString()); } } } } };
static Struct convertURLEntity(URLEntity uRLEntity) { return new Struct(SCHEMA_URL_ENTITY) .put("URL", uRLEntity.getURL()) .put("Text", uRLEntity.getText()) .put("ExpandedURL", uRLEntity.getExpandedURL()) .put("Start", uRLEntity.getStart()) .put("End", uRLEntity.getEnd()) .put("DisplayURL", uRLEntity.getDisplayURL()); }
for (URLEntity urle : status.getURLEntities()) { if(urle.getExpandedURL()){System.out.println(urle.getExpandedURL());} else {System.out.println(urle.getDisplayURL());} }
/** * Helper method to generate a URL-specialized {@link Token}. * * @param urlEntity The entity to map as a Token * * @return A token that correctly this entity. */ private Token linkOfEntity(final URLEntity urlEntity) { LOGGER.trace("Tokenizing URLEntity {}", urlEntity); return new Token( urlEntity.getDisplayURL(), urlEntity.getStart(), urlEntity.getEnd(), Token.TokenType.CLICKABLE, () -> browserSupport.openUrl(urlEntity.getExpandedURL()) ); }
if(urls[0].getExpandedURL() == null) { return null; String articleUrl = urls[0].getExpandedURL().toString(); Article article = null; try {
private String filterOutURLFromTweet(final Status status) { final String tweet = status.getText(); final URLEntity[] urlEntities = status.getURLEntities(); int startOfURL; int endOfURL; String truncatedTweet = ""; for(final URLEntity urlEntity: urlEntities){ startOfURL = urlEntity.getStart(); endOfURL = urlEntity.getEnd(); truncatedTweet += tweet.substring(0, startOfURL) + tweet.substring(endOfURL); } return truncatedTweet; }
@Override public void onStatus(Status status) { // add the tweet into the queue buffer //queue.offer(status.getText()); //add URLs to the queue buffer String url; if(status.getURLEntities().length != 0){ url = status.getURLEntities()[0].getURL(); queue.offer(url); } }
for (final URLEntity link : this.status.getURLEntities()) { final USMFStatus.Link l = new USMFStatus.Link(); l.href = link.getExpandedURL();
private void extractUrls(Status tweet, Set<Content> contents) { for (URLEntity urlEntity : tweet.getURLEntities()) { String url = urlEntity.getExpandedURL(); url = url == null ? urlEntity.getURL() : url; Content shareUrl = newBase(tweet); shareUrl.setContentName(url); shareUrl.setContentType("url"); contents.add(shareUrl); } }
/** * @return The list of {@link Token}s that cannot be built from {@link Status#getURLEntities()} alone. */ @Override public List<Token> extractTokens(final Status status) { final String statusText = status.getText(); final URLEntity[] managedUrlEntities = status.getURLEntities(); String unamanagedText = statusText; for (final URLEntity processedEntity : managedUrlEntities) { unamanagedText = unamanagedText.replaceAll(processedEntity.getURL(), ""); } return URLMatcher.findAllUrlsWithPosition(unamanagedText).stream().map( urlWithPos -> new Token( urlWithPos._1, urlWithPos._2, urlWithPos._3, Token.TokenType.CLICKABLE, () -> browserSupport.openUrl(urlWithPos._1) ) ).collect(Collectors.toList()); }
for (final URLEntity link : this.status.getURLEntities()) { final USMFStatus.Link l = new USMFStatus.Link(); l.href = link.getExpandedURL();
try { finalUrl = new URL(url.getExpandedURL()); if(!url.getURL().toString().equals(finalUrl.toString())) { extra = ", \"original\": \"" + url.getURL().toString() + "\""; msg.put("host", finalUrl.getHost()); if(extra != null) { msg.put("original", url.getURL().toString());
private void prepareEntities(Status status, Tweet.Builder tweetBuilder) { for (URLEntity urlEntity : status.getURLEntities()) { tweetBuilder.addUrlEntity(urlEntity.getURL(), urlEntity.getExpandedURL()); } for (MediaEntity mediaEntity : status.getMediaEntities()) { if (isImage(mediaEntity)) { tweetBuilder.addImageEntity(mediaEntity.getURL(), mediaEntity.getMediaURL()); } } }
String u = map.getExpandedURL(); u = map.getURL();
String u = map.getExpandedURL(); if (u == null) { u = map.getURL();
@Override public void emitTuples() { for (int size = statuses.size(); size-- > 0;) { Status s = statuses.poll(); if (status.isConnected()) { status.emit(s); } if (text.isConnected()) { text.emit(s.getText()); } if (url.isConnected()) { URLEntity[] entities = s.getURLEntities(); if (entities != null) { for (URLEntity ue : entities) { url.emit((ue.getExpandedURL() == null ? ue.getURL() : ue.getExpandedURL()).toString()); } } } if (hashtag.isConnected()) { HashtagEntity[] hashtagEntities = s.getHashtagEntities(); if (hashtagEntities != null) { for (HashtagEntity he : hashtagEntities) { hashtag.emit(he.getText()); } } } } }
@Override public void emitTuples() { for (int size = statuses.size(); size-- > 0;) { Status s = statuses.poll(); if (status.isConnected()) { status.emit(s); } if (text.isConnected()) { text.emit(s.getText()); } if (url.isConnected()) { URLEntity[] entities = s.getURLEntities(); if (entities != null) { for (URLEntity ue : entities) { url.emit((ue.getExpandedURL() == null ? ue.getURL() : ue.getExpandedURL()).toString()); } } } if (hashtag.isConnected()) { HashtagEntity[] hashtagEntities = s.getHashtagEntities(); if (hashtagEntities != null) { for (HashtagEntity he : hashtagEntities) { hashtag.emit(he.getText()); } } } } }