@Override protected AuditDto toAuditDto(AuditInfo auditInfo) { if ( auditInfo == null ) { return null; } AuditDto auditDto = new AuditDto(); auditDto.setCreatedDate( dateConverters.fromLongToDate( auditInfo.getCreatedDate() ) ); auditDto.setCreatedBy( auditInfo.getCreatedBy() ); auditDto.setLastModifiedDate( dateConverters.fromLongToDate( auditInfo.getLastUpdatedDate() ) ); auditDto.setLastModifiedBy( auditInfo.getLastUpdatedBy() ); return auditDto; }
@Override protected AuditInfo fromAuditDto(AuditDto audit) { if ( audit == null ) { return null; } AuditInfo auditInfo = new AuditInfo(); auditInfo.setLastUpdatedBy( audit.getLastModifiedBy() ); auditInfo.setLastUpdatedDate( dateConverters.fromDateToLong( audit.getLastModifiedDate() ) ); auditInfo.setCreatedDate( dateConverters.fromDateToLong( audit.getCreatedDate() ) ); auditInfo.setCreatedBy( audit.getCreatedBy() ); return auditInfo; }
/** * {@inheritDoc} */ @Override public PartitionDto hiveToMetacatPartition(final TableDto tableDto, final Partition partition) { final QualifiedName tableName = tableDto.getName(); final QualifiedName partitionName = QualifiedName.ofPartition(tableName.getCatalogName(), tableName.getDatabaseName(), tableName.getTableName(), getNameFromPartVals(tableDto, partition.getValues())); final PartitionDto result = new PartitionDto(); String owner = ""; if (tableDto.getSerde() != null) { owner = tableDto.getSerde().getOwner(); } //not setting Serde to view if (null == tableDto.getView() || Strings.isNullOrEmpty(tableDto.getView().getViewOriginalText())) { result.setSerde(toStorageDto(partition.getSd(), owner)); } result.setMetadata(partition.getParameters()); final AuditDto auditDto = new AuditDto(); auditDto.setCreatedDate(epochSecondsToDate(partition.getCreateTime())); auditDto.setLastModifiedDate(epochSecondsToDate(partition.getLastAccessTime())); result.setAudit(auditDto); result.setName(partitionName); return result; }
@Override public TableDto hiveToMetacatTable(final QualifiedName name, final Table table) { final TableDto dto = new TableDto(); dto.setSerde(toStorageDto(table.getSd(), table.getOwner())); dto.setAudit(new AuditDto()); dto.setName(name); if (table.isSetCreateTime()) { dto.getAudit().setCreatedDate(epochSecondsToDate(table.getCreateTime())); } dto.setMetadata(table.getParameters()); final List<FieldSchema> nonPartitionColumns = table.getSd().getCols(); final List<FieldSchema> partitionColumns = table.getPartitionKeys(); final List<FieldDto> allFields = Lists.newArrayListWithCapacity(nonPartitionColumns.size() + partitionColumns.size()); nonPartitionColumns.stream() .map(field -> this.hiveToMetacatField(field, false)) .forEachOrdered(allFields::add); partitionColumns.stream() .map(field -> this.hiveToMetacatField(field, true)) .forEachOrdered(allFields::add); dto.setFields(allFields); return dto; }
if (auditDto.getCreatedDate() != null) { result.setCreateTime(dateToEpochSeconds(auditDto.getCreatedDate())); if (auditDto.getLastModifiedDate() != null) { result.setLastAccessTime(dateToEpochSeconds(auditDto.getLastModifiedDate()));
if (auditDto != null && auditDto.getCreatedDate() != null) { table.setCreateTime(dateToEpochSeconds(auditDto.getCreatedDate()));
/** * Save all tables to index it in elastic search. * * @param databaseName database name * @param dtos table dtos * @return future */ private ListenableFuture<Void> indexTableDtos(final QualifiedName databaseName, final List<Optional<TableDto>> dtos) { return esService.submit(() -> { final List<ElasticSearchDoc> docs = dtos.stream().filter(dto -> dto != null && dto.isPresent()).map( tableDtoOptional -> { final TableDto dto = tableDtoOptional.get(); final String userName = dto.getAudit() != null ? dto.getAudit().getCreatedBy() : "admin"; return new ElasticSearchDoc(dto.getName().toString(), dto, userName, false, refreshMarkerText); }).collect(Collectors.toList()); log.info("Saving tables for database: {}", databaseName); elasticSearchUtil.save(ElasticSearchDoc.Type.table.name(), docs); return null; }); }
@Override public PartitionDto hiveToMetacatPartition(final TableDto tableDto, final Partition partition) { final QualifiedName tableName = tableDto.getName(); final QualifiedName partitionName = QualifiedName.ofPartition(tableName.getCatalogName(), tableName.getDatabaseName(), tableName.getTableName(), getNameFromPartVals(tableDto, partition.getValues())); final PartitionDto result = new PartitionDto(); String owner = ""; if (tableDto.getSerde() != null) { owner = tableDto.getSerde().getOwner(); } result.setSerde(toStorageDto(partition.getSd(), owner)); result.setMetadata(partition.getParameters()); final AuditDto auditDto = new AuditDto(); auditDto.setCreatedDate(epochSecondsToDate(partition.getCreateTime())); auditDto.setLastModifiedDate(epochSecondsToDate(partition.getLastAccessTime())); result.setAudit(auditDto); result.setName(partitionName); return result; }
/** * {@inheritDoc} */ @Override public TableDto hiveToMetacatTable(final QualifiedName name, final Table table) { final TableDto dto = new TableDto(); dto.setSerde(toStorageDto(table.getSd(), table.getOwner())); dto.setAudit(new AuditDto()); dto.setName(name); if (table.isSetCreateTime()) { dto.getAudit().setCreatedDate(epochSecondsToDate(table.getCreateTime())); } dto.setMetadata(table.getParameters()); final List<FieldSchema> nonPartitionColumns = table.getSd().getCols(); final List<FieldSchema> partitionColumns = table.getPartitionKeys(); final List<FieldDto> allFields = Lists.newArrayListWithCapacity(nonPartitionColumns.size() + partitionColumns.size()); nonPartitionColumns.stream() .map(field -> this.hiveToMetacatField(field, false)) .forEachOrdered(allFields::add); partitionColumns.stream() .map(field -> this.hiveToMetacatField(field, true)) .forEachOrdered(allFields::add); dto.setFields(allFields); dto.setView(new ViewDto(table.getViewOriginalText(), table.getViewExpandedText())); return dto; }
if (auditDto.getCreatedDate() != null) { result.setCreateTime(dateToEpochSeconds(auditDto.getCreatedDate())); if (auditDto.getLastModifiedDate() != null) { result.setLastAccessTime(dateToEpochSeconds(auditDto.getLastModifiedDate()));
if (auditDto != null && auditDto.getCreatedDate() != null) { table.setCreateTime(dateToEpochSeconds(auditDto.getCreatedDate()));
/** * Save all tables to index it in elastic search. * * @param tableName database name * @param dtos partition dtos * @return future */ private ListenableFuture<Void> indexPartitionDtos(final QualifiedName tableName, final List<PartitionDto> dtos) { return esService.submit(() -> { final List<ElasticSearchDoc> docs = dtos.stream().filter(dto -> dto != null).map( dto -> { final String userName = dto.getAudit() != null ? dto.getAudit().getCreatedBy() : "admin"; return new ElasticSearchDoc(dto.getName().toString(), dto, userName, false, refreshMarkerText); }).collect(Collectors.toList()); log.info("Saving partitions for tableName: {}", tableName); elasticSearchUtil.save(ElasticSearchDoc.Type.partition.name(), docs); return null; }); }
/** * {@inheritDoc} */ @Override public PartitionDto hiveToMetacatPartition(final TableDto tableDto, final Partition partition) { final QualifiedName tableName = tableDto.getName(); final QualifiedName partitionName = QualifiedName.ofPartition(tableName.getCatalogName(), tableName.getDatabaseName(), tableName.getTableName(), getNameFromPartVals(tableDto, partition.getValues())); final PartitionDto result = new PartitionDto(); String owner = ""; if (tableDto.getSerde() != null) { owner = tableDto.getSerde().getOwner(); } //not setting Serde to view if (null == tableDto.getView() || Strings.isNullOrEmpty(tableDto.getView().getViewOriginalText())) { result.setSerde(toStorageDto(partition.getSd(), owner)); } result.setMetadata(partition.getParameters()); final AuditDto auditDto = new AuditDto(); auditDto.setCreatedDate(epochSecondsToDate(partition.getCreateTime())); auditDto.setLastModifiedDate(epochSecondsToDate(partition.getLastAccessTime())); result.setAudit(auditDto); result.setName(partitionName); return result; }
/** * {@inheritDoc} */ @Override public TableDto hiveToMetacatTable(final QualifiedName name, final Table table) { final TableDto dto = new TableDto(); dto.setSerde(toStorageDto(table.getSd(), table.getOwner())); dto.setAudit(new AuditDto()); dto.setName(name); if (table.isSetCreateTime()) { dto.getAudit().setCreatedDate(epochSecondsToDate(table.getCreateTime())); } dto.setMetadata(table.getParameters()); final List<FieldSchema> nonPartitionColumns = table.getSd().getCols(); final List<FieldSchema> partitionColumns = table.getPartitionKeys(); final List<FieldDto> allFields = Lists.newArrayListWithCapacity(nonPartitionColumns.size() + partitionColumns.size()); nonPartitionColumns.stream() .map(field -> this.hiveToMetacatField(field, false)) .forEachOrdered(allFields::add); partitionColumns.stream() .map(field -> this.hiveToMetacatField(field, true)) .forEachOrdered(allFields::add); dto.setFields(allFields); dto.setView(new ViewDto(table.getViewOriginalText(), table.getViewExpandedText())); return dto; }
if (auditDto.getCreatedDate() != null) { result.setCreateTime(dateToEpochSeconds(auditDto.getCreatedDate())); if (auditDto.getLastModifiedDate() != null) { result.setLastAccessTime(dateToEpochSeconds(auditDto.getLastModifiedDate()));
if (auditDto != null && auditDto.getCreatedDate() != null) { table.setCreateTime(dateToEpochSeconds(auditDto.getCreatedDate()));