/** * As Google Drive extractor needs file system helper, it invokes to initialize file system helper. * {@inheritDoc} * @see org.apache.gobblin.source.Source#getExtractor(org.apache.gobblin.configuration.WorkUnitState) */ @Override public Extractor<S, D> getExtractor(WorkUnitState state) throws IOException { Preconditions.checkNotNull(state, "WorkUnitState should not be null"); LOG.info("WorkUnitState from getExtractor: " + state); try { //GoogleDriveExtractor needs GoogleDriveFsHelper initFileSystemHelper(state); } catch (FileBasedHelperException e) { throw new IOException(e); } Preconditions.checkNotNull(fsHelper, "File system helper should not be null"); return new GoogleDriveExtractor<>(state, fsHelper); }
private static void checkRowLimit(int rowLimit) { Preconditions.checkArgument(rowLimit > 0 && rowLimit <= API_ROW_LIMIT, "Row limit for Google Search Console API must be within range (0, 5000]"); } }
@Override public void unbzip2Project( String projectName, InputStream dataStream ) throws IOException { Preconditions.checkArgument( Project.isValidProjectName(projectName), "[%s] invalid project name: ", projectName ); Preconditions.checkState( getDirForProject(projectName).mkdirs(), "[%s] directories for " + "evicted project already exist", projectName ); Tar.bz2.unzip(dataStream, getDirForProject(projectName)); }
private static File doRename(File fileOrDir, String from, String to) { if (!fileOrDir.getName().equals(from)) { return fileOrDir; } File renamed = new File(fileOrDir.getParent(), to); Preconditions.checkState(fileOrDir.renameTo(renamed)); return renamed; }
Preconditions.checkState(initialized, "GmailSyncer.init() must be called first"); Multimap<LocalMessage, Message> map = mailbox.mapMessageIds(messages);
/** * Initialize file system helper at most once for this instance. * {@inheritDoc} * @see org.apache.gobblin.source.extractor.filebased.FileBasedSource#initFileSystemHelper(org.apache.gobblin.configuration.State) */ @Override public synchronized void initFileSystemHelper(State state) throws FileBasedHelperException { if (fsHelper == null) { Credential credential = new GoogleCommon.CredentialBuilder(state.getProp(SOURCE_CONN_PRIVATE_KEY), state.getPropAsList(API_SCOPES)) .fileSystemUri(state.getProp(PRIVATE_KEY_FILESYSTEM_URI)) .proxyUrl(state.getProp(SOURCE_CONN_USE_PROXY_URL)) .port(state.getProp(SOURCE_CONN_USE_PROXY_PORT)) .serviceAccountId(state.getProp(SOURCE_CONN_USERNAME)) .build(); Drive driveClient = new Drive.Builder(credential.getTransport(), GoogleCommon.getJsonFactory(), credential) .setApplicationName(Preconditions.checkNotNull(state.getProp(APPLICATION_NAME), "ApplicationName is required")) .build(); this.fsHelper = closer.register(new GoogleDriveFsHelper(state, driveClient)); } }
public UrlTriePostOrderIterator(UrlTrie trie, int stoppingSize) { Preconditions.checkArgument(stoppingSize > 0); _currentNode = trie.getRoot(); String prefix = trie.getPrefix(); _currentPrefixSb = new StringBuilder(); if (prefix != null) { _currentPrefixSb.append(prefix); } _groupSize = stoppingSize; }
@Override public void onSuccess( Message responseMessage, HttpHeaders responseHeaders) throws IOException { Preconditions.checkState( message.getId().equals(responseMessage.getId()), "Message ids must be equal"); List<String> gmailMessageIds = responseMessage.getLabelIds() == null ? ImmutableList.of() : responseMessage.getLabelIds(); System.out.format("For message %s, got labels: %s\n", responseMessage.getId(), gmailMessageIds.stream() .map(id -> labelsById.getOrDefault( id, new Label().setName(id))) .map(Label::getName) .collect(Collectors.joining(", "))); message.setLabelIds(gmailMessageIds); } });
/** * Sets a number of the project. */ Builder setNumber(BigInteger number) { this.number = checkNotNull(number); return this; }
wuState.getPropAsInt(GoggleIngestionConfigurationKeys.SOURCE_ASYNC_ITERATOR_POLL_BLOCKING_TIME, 1)); _wuState = wuState; Preconditions.checkArgument(!filterMap.containsKey(GoogleWebmasterFilter.Dimension.PAGE), "Doesn't support filters for page for the time being. Will implement support later. If page filter is provided, the code won't take the responsibility of get all pages, so it will just return all queries for that page."); Preconditions.checkArgument(PAGE_LIMIT >= 1, "Page limit must be at least 1."); Preconditions.checkArgument(QUERY_LIMIT >= 1, "Query limit must be at least 1."); Preconditions.checkArgument(ROUND_TIME_OUT > 0, "Time out must be positive."); Preconditions.checkArgument(MAX_RETRY_ROUNDS >= 0, "Retry rounds cannot be negative."); Preconditions.checkArgument(ROUND_COOL_DOWN >= 0, "Initial cool down time cannot be negative."); Preconditions.checkArgument(batchesPerSecond > 0, "Requests per second must be positive."); Preconditions.checkArgument(BATCH_SIZE >= 1, "Batch size must be at least 1."); Preconditions.checkArgument(TRIE_GROUP_SIZE >= 1, "Group size must be at least 1."); Preconditions.checkArgument(PAGE_LIMIT == GoogleWebmasterClient.API_ROW_LIMIT, "Page limit must be set at 5000 if you want to use the advanced algorithm. This indicates that you understand what you are doing.");
/** * Sets a number of the project. */ Builder number(BigInteger number) { this.number = checkNotNull(number); return this; }
@Override public List<String> getPages(String siteProperty, String startDate, String endDate, String country, int rowLimit, List<GoogleWebmasterFilter.Dimension> requestedDimensions, List<ApiDimensionFilter> filters, int startRow) throws IOException { checkRowLimit(rowLimit); Preconditions.checkArgument(requestedDimensions.contains(GoogleWebmasterFilter.Dimension.PAGE)); SearchAnalyticsQueryResponse rspByCountry = createSearchAnalyticsQuery(siteProperty, startDate, endDate, requestedDimensions, GoogleWebmasterFilter.andGroupFilters(filters), rowLimit, startRow).execute(); List<ApiDataRow> pageRows = rspByCountry.getRows(); List<String> pages = new ArrayList<>(rowLimit); if (pageRows != null) { int pageIndex = requestedDimensions.indexOf(GoogleWebmasterFilter.Dimension.PAGE); for (ApiDataRow row : pageRows) { pages.add(row.getKeys().get(pageIndex)); } } return pages; }
/** * Sets an id of the project. */ Builder setId(String id) { this.id = checkNotNull(id); return this; }
GoogleWebmasterDataFetcherImpl(String siteProperty, GoogleWebmasterClient client, State wuState) throws IOException { _siteProperty = siteProperty; Preconditions.checkArgument(_siteProperty.endsWith("/"), "The site property must end in \"/\""); _client = client; _jobs = getHotStartJobs(wuState); API_REQUESTS_PER_SECOND = wuState.getPropAsDouble(GoogleWebMasterSource.KEY_PAGES_TUNING_REQUESTS_PER_SECOND, 4.5); PAGES_COUNT_COOLDOWN_TIME = wuState.getPropAsInt(GoogleWebMasterSource.KEY_PAGES_COUNT_TUNING_COOLDOWN_TIME, 30); PAGES_GET_COOLDOWN_TIME = wuState.getPropAsInt(GoogleWebMasterSource.KEY_PAGES_GET_TUNING_COOLDOWN_TIME, 5); LIMITER = new RateBasedLimiter(API_REQUESTS_PER_SECOND, TimeUnit.SECONDS); GET_PAGES_RETRIES = wuState.getPropAsInt(GoogleWebMasterSource.KEY_PAGES_TUNING_MAX_RETRIES, 120); }
/** * Sets quotas assigned to the project. */ Builder setQuota(Quota quota) { this.quota = checkNotNull(quota); return this; }
/** * Performs all necessary setup steps for running requests against the API. * * @param applicationName the name of the application: com.example.app * @param serviceAccountEmail the Service Account Email (empty if using * installed application) * @return the {@Link AndroidPublisher} service * @throws GeneralSecurityException * @throws IOException */ public static AndroidPublisher init( String applicationName, @Nullable String serviceAccountEmail ) throws IOException, GeneralSecurityException { Preconditions.checkArgument( !Strings.isNullOrEmpty( applicationName ), "applicationName cannot be null or empty!" ); // Authorization. newTrustedTransport(); Credential credential; credential = authorizeWithServiceAccount( serviceAccountEmail ); // Set up and return API client. return new AndroidPublisher.Builder( httpTransport, JSON_FACTORY, credential ).setApplicationName( applicationName ) .build(); }
/** * Sets quotas assigned to the project. */ Builder quota(Quota quota) { this.quota = checkNotNull(quota); return this; }
private static void checkFileSize(long size) { Preconditions.checkArgument( size >= 0 && size <= Integer.MAX_VALUE, "file too big (" + size + " B): " + "tarTo should have thrown an IOException" ); }
/** * Sets an id of the project. */ Builder id(String id) { this.id = checkNotNull(id); return this; }
private static void checkRowLimit(int rowLimit) { Preconditions.checkArgument(rowLimit > 0 && rowLimit <= API_ROW_LIMIT, "Row limit for Google Search Console API must be within range (0, 5000]"); } }