private PartitionDto getPartitionDtoByName(final TableDto tableDto, final String partName) throws TException { final GetPartitionsRequestDto dto = new GetPartitionsRequestDto(null, ImmutableList.of(partName), true, false); final List<PartitionDto> partitionDtos = partV1.getPartitionsForRequest( catalogName, tableDto.getName().getDatabaseName(), tableDto.getName().getTableName(), null, null, null, null, false, dto); if (partitionDtos == null || partitionDtos.isEmpty()) { throw new NoSuchObjectException("Partition (" + partName + ") not found on " + tableDto.getName()); } else if (partitionDtos.size() != 1) { // I don't think this is even possible throw new NoSuchObjectException("Partition (" + partName + ") matched extra on " + tableDto.getName()); } return partitionDtos.get(0); }
private PartitionDto getPartitionDtoByName(final TableDto tableDto, final String partName) throws TException { final GetPartitionsRequestDto dto = new GetPartitionsRequestDto(null, ImmutableList.of(partName), true, false); final List<PartitionDto> partitionDtos = partV1.getPartitionsForRequest( catalogName, tableDto.getName().getDatabaseName(), tableDto.getName().getTableName(), null, null, null, null, false, dto); if (partitionDtos == null || partitionDtos.isEmpty()) { throw new NoSuchObjectException("Partition (" + partName + ") not found on " + tableDto.getName()); } else if (partitionDtos.size() != 1) { // I don't think this is even possible throw new NoSuchObjectException("Partition (" + partName + ") matched extra on " + tableDto.getName()); } return partitionDtos.get(0); }
private void updateEntitiesWithSameUri(final String metadataType, final TableDto dto, final String userName) { if (dto.isDataExternal()) { final List<String> ids = es.getTableIdsByUri(metadataType, dto.getDataUri()) .stream().filter(s -> !s.equals(dto.getName().toString())).collect(Collectors.toList()); if (!ids.isEmpty()) { log.info("ElasticSearch table updates({}) with same uri {} (Table:{})", ids.size(), dto.getDataUri(), dto.getName()); final ObjectNode node = metacatJsonLocator.emptyObjectNode(); node.set(ElasticSearchDoc.Field.DATA_METADATA, dto.getDataMetadata()); node.put(ElasticSearchDoc.Field.USER, userName); node.put(ElasticSearchDoc.Field.TIMESTAMP, java.time.Instant.now().toEpochMilli()); es.updates(ElasticSearchDoc.Type.table.name(), ids, node); } } }
throws MetaException { if (partVals.size() > tableDto.getPartition_keys().size()) { throw new MetaException("Too many partition values for " + tableDto.getName());
/** * Save all tables to index it in elastic search. * * @param databaseName database name * @param dtos table dtos * @return future */ private ListenableFuture<Void> indexTableDtos(final QualifiedName databaseName, final List<Optional<TableDto>> dtos) { return esService.submit(() -> { final List<ElasticSearchDoc> docs = dtos.stream().filter(dto -> dto != null && dto.isPresent()).map( tableDtoOptional -> { final TableDto dto = tableDtoOptional.get(); final String userName = dto.getAudit() != null ? dto.getAudit().getCreatedBy() : "admin"; return new ElasticSearchDoc(dto.getName().toString(), dto, userName, false, refreshMarkerText); }).collect(Collectors.toList()); log.info("Saving tables for database: {}", databaseName); elasticSearchUtil.save(ElasticSearchDoc.Type.table.name(), docs); return null; }); }
private List<Partition> addPartitionsCore(final String dbName, final String tblName, final List<Partition> parts, final boolean ifNotExists) throws TException { log.debug("Ignoring {} since metacat save partitions will do an update if it already exists", ifNotExists); final TableDto tableDto = v1.getTable(catalogName, dbName, tblName, true, false, false); final List<String> partitionKeys = tableDto.getPartition_keys(); if (partitionKeys == null || partitionKeys.isEmpty()) { throw new MetaException("Unable to add partition to unpartitioned table: " + tableDto.getName()); } final PartitionsSaveRequestDto partitionsSaveRequestDto = new PartitionsSaveRequestDto(); final List<PartitionDto> converted = Lists.newArrayListWithCapacity(parts.size()); for (Partition partition : parts) { converted.add(hiveConverters.hiveToMetacatPartition(tableDto, partition)); } partitionsSaveRequestDto.setPartitions(converted); partV1.savePartitions(catalogName, dbName, tblName, partitionsSaveRequestDto); return parts; }
private List<Partition> addPartitionsCore(final String dbName, final String tblName, final List<Partition> parts, final boolean ifNotExists) throws TException { log.debug("Ignoring {} since metacat save partitions will do an update if it already exists", ifNotExists); final TableDto tableDto = v1.getTable(catalogName, dbName, tblName, true, false, false); final List<String> partitionKeys = tableDto.getPartition_keys(); if (partitionKeys == null || partitionKeys.isEmpty()) { throw new MetaException("Unable to add partition to unpartitioned table: " + tableDto.getName()); } final PartitionsSaveRequestDto partitionsSaveRequestDto = new PartitionsSaveRequestDto(); final List<PartitionDto> converted = Lists.newArrayListWithCapacity(parts.size()); for (Partition partition : parts) { converted.add(hiveConverters.hiveToMetacatPartition(tableDto, partition)); } partitionsSaveRequestDto.setPartitions(converted); partV1.savePartitions(catalogName, dbName, tblName, partitionsSaveRequestDto); return parts; }
/** * {@inheritDoc} */ @Override public boolean drop_partition_with_environment_context( final String dbName, final String tblName, final List<String> partVals, final boolean deleteData, @Nullable final EnvironmentContext environmentContext ) throws TException { return requestWrapper("drop_partition_with_environment_context", new Object[]{dbName, tblName, partVals, deleteData, environmentContext}, () -> { final TableDto tableDto = getTableDto(dbName, tblName); final String partName = hiveConverters.getNameFromPartVals(tableDto, partVals); final QualifiedName partitionName = getPartitionDtoByName(tableDto, partName).getName(); if (deleteData) { log.warn("Ignoring command to delete data for {}/{}/{}/{}", catalogName, tableDto.getName().getDatabaseName(), tableDto.getName().getTableName(), partitionName.getPartitionName()); } partV1.deletePartitions( catalogName, tableDto.getName().getDatabaseName(), tableDto.getName().getTableName(), ImmutableList.of(partitionName.getPartitionName())); return true; }); }
/** * {@inheritDoc} */ @Override public boolean drop_partition_with_environment_context( final String dbName, final String tblName, final List<String> partVals, final boolean deleteData, @Nullable final EnvironmentContext environmentContext ) throws TException { return requestWrapper("drop_partition_with_environment_context", new Object[]{dbName, tblName, partVals, deleteData, environmentContext}, () -> { final TableDto tableDto = getTableDto(dbName, tblName); final String partName = hiveConverters.getNameFromPartVals(tableDto, partVals); final QualifiedName partitionName = getPartitionDtoByName(tableDto, partName).getName(); if (deleteData) { log.warn("Ignoring command to delete data for {}/{}/{}/{}", catalogName, tableDto.getName().getDatabaseName(), tableDto.getName().getTableName(), partitionName.getPartitionName()); } partV1.deletePartitions( catalogName, tableDto.getName().getDatabaseName(), tableDto.getName().getTableName(), ImmutableList.of(partitionName.getPartitionName())); return true; }); }
/** * Subscriber. * * @param event event */ @EventListener public void metacatDeleteTablePostEventHandler(final MetacatDeleteTablePostEvent event) { if (config.canCascadeViewsMetadataOnTableDelete() && !event.isMView()) { final QualifiedName name = event.getTable().getName(); try { // delete views associated with this table final List<NameDateDto> viewNames = mViewService.list(name); viewNames.forEach(viewName -> mViewService.deleteAndReturn(viewName.getName())); } catch (Exception e) { log.warn("Failed cleaning mviews after deleting table {}", name); } // delete table partitions metadata try { final List<QualifiedName> names = userMetadataService.getDescendantDefinitionNames(name); if (names != null && !names.isEmpty()) { userMetadataService.deleteDefinitionMetadata(names); } } catch (Exception e) { log.warn("Failed cleaning partition definition metadata after deleting table {}", name); } } }
/** * Subscriber. * * @param event event */ @EventListener public void metacatDeleteTablePostEventHandler(final MetacatDeleteTablePostEvent event) { log.debug("Received DeleteTableEvent {}", event); this.tableDeleteEventsDelayTimer .record(System.currentTimeMillis() - event.getRequestContext().getTimestamp(), TimeUnit.MILLISECONDS); this.tableDeleteTimer.record(() -> { final TableDto dto = event.getTable(); es.softDelete(ElasticSearchDoc.Type.table.name(), dto.getName().toString(), event.getRequestContext()); if (config.isElasticSearchPublishPartitionEnabled()) { try { final List<String> partitionIdsToBeDeleted = es.getIdsByQualifiedName(ElasticSearchDoc.Type.partition.name(), dto.getName()); es.delete(ElasticSearchDoc.Type.partition.name(), partitionIdsToBeDeleted); } catch (Exception e) { log.warn("Failed deleting the partitions for the dropped table/view:{}", dto.getName()); } } }); }
/** * Subscriber. * * @param event event */ @EventListener public void metacatRenameTablePostEventHandler(final MetacatRenameTablePostEvent event) { if (!event.isMView()) { final QualifiedName oldName = event.getOldTable().getName(); final QualifiedName newName = event.getCurrentTable().getName(); final List<NameDateDto> views = mViewService.list(oldName); if (views != null && !views.isEmpty()) { views.forEach(view -> { final QualifiedName newViewName = QualifiedName .ofView(oldName.getCatalogName(), oldName.getDatabaseName(), newName.getTableName(), view.getName().getViewName()); mViewService.rename(view.getName(), newViewName); }); } } } }
/** * Subscriber. * * @param event event */ @EventListener public void metacatCreateTablePostEventHandler(final MetacatCreateTablePostEvent event) { log.debug("Received CreateTableEvent {}", event); this.tableCreateEventsDelayTimer .record(System.currentTimeMillis() - event.getRequestContext().getTimestamp(), TimeUnit.MILLISECONDS); this.tableCreateTimer.record(() -> { final TableDto dto = event.getTable(); final ElasticSearchDoc doc = new ElasticSearchDoc(dto.getName().toString(), dto, event.getRequestContext().getUserName(), false); es.save(ElasticSearchDoc.Type.table.name(), doc.getId(), doc); }); }
/** * Subscriber. * * @param event event */ @EventListener public void metacatRenameTablePostEventHandler(final MetacatRenameTablePostEvent event) { log.debug("Received RenameTableEvent {}", event); this.tableRenameEventsDelayTimer .record(System.currentTimeMillis() - event.getRequestContext().getTimestamp(), TimeUnit.MILLISECONDS); this.tableRenameTimer.record(() -> { es.delete(ElasticSearchDoc.Type.table.name(), event.getName().toString()); final TableDto dto = event.getCurrentTable(); final ElasticSearchDoc doc = new ElasticSearchDoc(dto.getName().toString(), dto, event.getRequestContext().getUserName(), false); es.save(ElasticSearchDoc.Type.table.name(), doc.getId(), doc); }); }
protected ConnectorTableMetadata fromTableDto(final TableDto table, final TypeManager typeManager) { return new ConnectorTableDetailMetadata( toSchemaTableName(table.getName()), columnsFromTable(table, typeManager), table.getSerde() == null ? null : table.getSerde().getOwner(), fromStorageDto(table.getSerde()), table.getMetadata(), fromAuditDto(table.getAudit()) ); }
/** * Subscriber. * * @param event event */ @EventListener public void metacatUpdateTablePostEventHandler(final MetacatUpdateTablePostEvent event) { log.debug("Received UpdateTableEvent {}", event); this.tableUpdateEventsDelayTimer .record(System.currentTimeMillis() - event.getRequestContext().getTimestamp(), TimeUnit.MILLISECONDS); this.tableUpdateTimer.record(() -> { final TableDto dto = event.getCurrentTable(); final ElasticSearchDoc doc = new ElasticSearchDoc(dto.getName().toString(), dto, event.getRequestContext().getUserName(), false); final ElasticSearchDoc oldDoc = es.get(ElasticSearchDoc.Type.table.name(), doc.getId()); es.save(ElasticSearchDoc.Type.table.name(), doc.getId(), doc); if (config.isElasticSearchUpdateTablesWithSameUriEnabled() && (oldDoc == null || oldDoc.getDto() == null || !Objects.equals(((TableDto) oldDoc.getDto()).getDataMetadata(), dto.getDataMetadata()))) { updateEntitiesWithSameUri(ElasticSearchDoc.Type.table.name(), dto, event.getRequestContext().getUserName()); } }); }
validate(name); this.authorizationService.checkPermission(metacatRequestContext.getUserName(), tableDto.getName(), MetacatOperation.CREATE);
/** * {@inheritDoc} */ @Override public PartitionDto hiveToMetacatPartition(final TableDto tableDto, final Partition partition) { final QualifiedName tableName = tableDto.getName(); final QualifiedName partitionName = QualifiedName.ofPartition(tableName.getCatalogName(), tableName.getDatabaseName(), tableName.getTableName(), getNameFromPartVals(tableDto, partition.getValues())); final PartitionDto result = new PartitionDto(); String owner = ""; if (tableDto.getSerde() != null) { owner = tableDto.getSerde().getOwner(); } //not setting Serde to view if (null == tableDto.getView() || Strings.isNullOrEmpty(tableDto.getView().getViewOriginalText())) { result.setSerde(toStorageDto(partition.getSd(), owner)); } result.setMetadata(partition.getParameters()); final AuditDto auditDto = new AuditDto(); auditDto.setCreatedDate(epochSecondsToDate(partition.getCreateTime())); auditDto.setLastModifiedDate(epochSecondsToDate(partition.getLastAccessTime())); result.setAudit(auditDto); result.setName(partitionName); return result; }
/** * {@inheritDoc} */ @Override public PartitionDto hiveToMetacatPartition(final TableDto tableDto, final Partition partition) { final QualifiedName tableName = tableDto.getName(); final QualifiedName partitionName = QualifiedName.ofPartition(tableName.getCatalogName(), tableName.getDatabaseName(), tableName.getTableName(), getNameFromPartVals(tableDto, partition.getValues())); final PartitionDto result = new PartitionDto(); String owner = ""; if (tableDto.getSerde() != null) { owner = tableDto.getSerde().getOwner(); } //not setting Serde to view if (null == tableDto.getView() || Strings.isNullOrEmpty(tableDto.getView().getViewOriginalText())) { result.setSerde(toStorageDto(partition.getSd(), owner)); } result.setMetadata(partition.getParameters()); final AuditDto auditDto = new AuditDto(); auditDto.setCreatedDate(epochSecondsToDate(partition.getCreateTime())); auditDto.setLastModifiedDate(epochSecondsToDate(partition.getLastAccessTime())); result.setAudit(auditDto); result.setName(partitionName); return result; }
@Override public PartitionDto hiveToMetacatPartition(final TableDto tableDto, final Partition partition) { final QualifiedName tableName = tableDto.getName(); final QualifiedName partitionName = QualifiedName.ofPartition(tableName.getCatalogName(), tableName.getDatabaseName(), tableName.getTableName(), getNameFromPartVals(tableDto, partition.getValues())); final PartitionDto result = new PartitionDto(); String owner = ""; if (tableDto.getSerde() != null) { owner = tableDto.getSerde().getOwner(); } result.setSerde(toStorageDto(partition.getSd(), owner)); result.setMetadata(partition.getParameters()); final AuditDto auditDto = new AuditDto(); auditDto.setCreatedDate(epochSecondsToDate(partition.getCreateTime())); auditDto.setLastModifiedDate(epochSecondsToDate(partition.getLastAccessTime())); result.setAudit(auditDto); result.setName(partitionName); return result; }