diff --git a/multiapps-controller-persistence/pom.xml b/multiapps-controller-persistence/pom.xml index 7e605c5525..e2a1ccb644 100644 --- a/multiapps-controller-persistence/pom.xml +++ b/multiapps-controller-persistence/pom.xml @@ -95,8 +95,12 @@ aws-s3 - org.apache.jclouds.provider - azureblob + com.azure + azure-storage-blob + + + com.azure + azure-core-http-okhttp org.apache.jclouds diff --git a/multiapps-controller-persistence/src/main/java/module-info.java b/multiapps-controller-persistence/src/main/java/module-info.java index eed9627c09..7c49523d76 100644 --- a/multiapps-controller-persistence/src/main/java/module-info.java +++ b/multiapps-controller-persistence/src/main/java/module-info.java @@ -28,6 +28,9 @@ requires transitive org.cloudfoundry.multiapps.controller.api; requires aliyun.sdk.oss; + requires com.azure.core; + requires com.azure.core.http.okhttp; + requires com.azure.storage.blob; requires com.fasterxml.jackson.annotation; requires com.fasterxml.jackson.databind; requires com.google.auth; diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java index 8d3a837c67..2ff0ed4ce8 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java @@ -54,6 +54,7 @@ public final class Messages { // ERROR log messages: public static final String UPLOAD_STREAM_FAILED_TO_CLOSE = "Cannot close file upload stream"; + public static final String CANNOT_PARSE_CONTAINER_URI_OF_OBJECT_STORE = "Cannot parse container_uri of object store"; // WARN log messages: public static final String COULD_NOT_CLOSE_RESULT_SET = "Could not close result set."; diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java new file mode 100644 index 0000000000..c4aadfa61e --- /dev/null +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java @@ -0,0 +1,215 @@ +package org.cloudfoundry.multiapps.controller.persistence.services; + +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import com.azure.core.http.HttpClient; +import com.azure.core.http.okhttp.OkHttpAsyncHttpClientBuilder; +import com.azure.core.http.policy.ExponentialBackoffOptions; +import com.azure.core.http.policy.RetryOptions; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.BlobServiceClient; +import com.azure.storage.blob.BlobServiceClientBuilder; +import com.azure.storage.blob.models.BlobItem; +import com.azure.storage.blob.models.BlobListDetails; +import com.azure.storage.blob.models.BlobRange; +import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.models.ListBlobsOptions; +import com.azure.storage.blob.options.BlobParallelUploadOptions; +import org.cloudfoundry.multiapps.controller.persistence.Messages; +import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; +import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; +import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; + +public class AzureObjectStoreFileStorage implements FileStorage { + + private static final String SAS_TOKEN = "sas_token"; + private static final String CONTAINER_NAME = "container_name"; + private static final String CONTAINER_URI = "container_uri"; + private final HttpClient httpClient; + private final BlobContainerClient containerClient; + + public AzureObjectStoreFileStorage(Map credentials) { + this.containerClient = createContainerClient(credentials); + this.httpClient = new OkHttpAsyncHttpClientBuilder().build(); + } + + @Override + public void addFile(FileEntry fileEntry, InputStream content) throws FileStorageException { + BlobClient blobClient = containerClient.getBlobClient(fileEntry.getId()); + try { + BlobParallelUploadOptions blobParallelUploadOptions = new BlobParallelUploadOptions(content); + blobParallelUploadOptions.setMetadata(ObjectStoreMapper.createFileEntryMetadata(fileEntry)); + + blobClient.uploadWithResponse(blobParallelUploadOptions, ObjectStoreConstants.OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES, + null); + } catch (BlobStorageException e) { + throw new FileStorageException(e); + } + } + + @Override + public List getFileEntriesWithoutContent(List fileEntries) throws FileStorageException { + Set existingFiles = getAllEntriesNames(); + return fileEntries.stream() + .filter(fileEntry -> !existingFiles.contains(fileEntry.getId())) + .toList(); + } + + @Override + public void deleteFile(String id, String space) throws FileStorageException { + BlobClient blobClient = containerClient.getBlobClient(id); + try { + blobClient.deleteIfExists(); + } catch (BlobStorageException e) { + throw new FileStorageException(e); + } + } + + @Override + public void deleteFilesBySpaceIds(List spaceIds) throws FileStorageException { + removeBlobsByFilter(blob -> ObjectStoreFilter.filterBySpaceIds(blob.getMetadata(), spaceIds)); + } + + @Override + public void deleteFilesBySpaceAndNamespace(String space, String namespace) { + removeBlobsByFilter(blob -> ObjectStoreFilter.filterBySpaceAndNamespace(blob.getMetadata(), space, namespace)); + } + + @Override + public int deleteFilesModifiedBefore(LocalDateTime modificationTime) throws FileStorageException { + return removeBlobsByFilter( + blob -> ObjectStoreFilter.filterByModificationTime(blob.getMetadata(), blob.getName(), modificationTime)); + } + + @Override + public T processFileContent(String space, String id, FileContentProcessor fileContentProcessor) throws FileStorageException { + FileEntry fileEntry = ObjectStoreMapper.createFileEntry(space, id); + try (InputStream inputStream = openBlobInputStream(fileEntry)) { + return fileContentProcessor.process(inputStream); + } catch (Exception e) { + throw new FileStorageException(e); + } + } + + private InputStream openBlobInputStream(FileEntry fileEntry) throws FileStorageException { + BlobClient blobClient = containerClient.getBlobClient(fileEntry.getId()); + try { + return blobClient.openInputStream(); + } catch (BlobStorageException e) { + throw new FileStorageException(e); + } + } + + @Override + public InputStream openInputStream(String space, String id) throws FileStorageException { + FileEntry fileEntry = ObjectStoreMapper.createFileEntry(space, id); + return openBlobInputStream(fileEntry); + } + + @Override + public void testConnection() { + containerClient.getBlobClient("test"); + } + + @Override + public void deleteFilesByIds(List fileIds) throws FileStorageException { + removeBlobsByFilter(blob -> fileIds.contains(blob.getName())); + } + + @Override + public T processArchiveEntryContent(FileContentToProcess fileContentToProcess, FileContentProcessor fileContentProcessor) + throws FileStorageException { + FileEntry fileEntry = ObjectStoreMapper.createFileEntry(fileContentToProcess.getSpaceGuid(), fileContentToProcess.getGuid()); + BlobClient blobClient = containerClient.getBlobClient(fileEntry.getId()); + long contentSize = fileContentToProcess.getEndOffset() - fileContentToProcess.getStartOffset(); + BlobRange blobRange = new BlobRange(fileContentToProcess.getStartOffset(), contentSize); + + try { + return fileContentProcessor.process(blobClient.openInputStream(blobRange, null)); + } catch (IOException e) { + throw new FileStorageException(e); + } + } + + protected BlobContainerClient createContainerClient(Map credentials) { + BlobServiceClient serviceClient = new BlobServiceClientBuilder().endpoint(getContainerUriEndpoint(credentials)) + .retryOptions(createRetryOptions()) + .httpClient(httpClient) + .sasToken((String) credentials.get(SAS_TOKEN)) + .buildClient(); + + return serviceClient.getBlobContainerClient((String) credentials.get(CONTAINER_NAME)); + } + + public String getContainerUriEndpoint(Map credentials) { + if (!credentials.containsKey(CONTAINER_URI)) { + return null; + } + try { + URL containerUri = new URL((String) credentials.get(CONTAINER_URI)); + return new URL(containerUri.getProtocol(), containerUri.getHost(), containerUri.getPort(), "").toString(); + } catch (MalformedURLException e) { + throw new IllegalStateException(Messages.CANNOT_PARSE_CONTAINER_URI_OF_OBJECT_STORE, e); + } + } + + private RetryOptions createRetryOptions() { + ExponentialBackoffOptions exponentialBackoffOptions = new ExponentialBackoffOptions().setBaseDelay( + ObjectStoreConstants.OBJECT_STORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS) + .setMaxDelay( + ObjectStoreConstants.OBJECT_STORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS) + .setMaxRetries( + ObjectStoreConstants.OBJECT_STORE_MAX_ATTEMPTS_CONFIG); + + return new RetryOptions(exponentialBackoffOptions); + } + + private int removeBlobsByFilter(Predicate filter) { + Set blobNames = getEntryNames(filter); + List deletedBlobsResults = new ArrayList<>(); + + if (blobNames.isEmpty()) { + return 0; + } + for (String blobName : blobNames) { + BlobClient blobClient = containerClient.getBlobClient(blobName); + deletedBlobsResults.add(blobClient.deleteIfExists()); + } + + deletedBlobsResults.removeIf(Boolean.FALSE::equals); + + return deletedBlobsResults.size(); + } + + protected Set getEntryNames(Predicate filter) { + ListBlobsOptions listBlobsOptions = new ListBlobsOptions(); + BlobListDetails blobListDetails = new BlobListDetails(); + blobListDetails.setRetrieveMetadata(true); + listBlobsOptions.setDetails(blobListDetails); + + return containerClient.listBlobs(listBlobsOptions, ObjectStoreConstants.OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES) + .stream() + .filter(filter) + .map(BlobItem::getName) + .collect(Collectors.toSet()); + } + + public Set getAllEntriesNames() { + return containerClient.listBlobs() + .stream() + .map(BlobItem::getName) + .collect(Collectors.toSet()); + } +} diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java index ea9c127ce2..153a734814 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java @@ -5,7 +5,6 @@ import java.io.InputStream; import java.nio.channels.Channels; import java.text.MessageFormat; -import java.time.Duration; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Base64; @@ -28,6 +27,7 @@ import com.google.cloud.storage.StorageRetryStrategy; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; import org.springframework.http.MediaType; @@ -36,13 +36,8 @@ public class GcpObjectStoreFileStorage implements FileStorage { private final String bucketName; private final Storage storage; - private static final String BUCKET = "bucket"; - private static final int OBJECT_STORE_MAX_ATTEMPTS_CONFIG = 6; - private static final double OBJECT_STORE_RETRY_DELAY_MULTIPLIER_CONFIG = 2.0; - private static final Duration OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES = Duration.ofMinutes(10); - private static final Duration OBJECT_STORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS = Duration.ofSeconds(10); - private static final Duration OBJECT_STORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS = Duration.ofMillis(250); private static final String BASE_64_ENCODED_PRIVATE_KEY_DATA = "base64EncodedPrivateKeyData"; + private static final String BUCKET = "bucket"; public GcpObjectStoreFileStorage(Map credentials) { this.bucketName = (String) credentials.get(BUCKET); @@ -55,11 +50,12 @@ protected Storage createObjectStoreStorage(Map credentials) { .setStorageRetryStrategy(StorageRetryStrategy.getUniformStorageRetryStrategy()) .setRetrySettings( RetrySettings.newBuilder() - .setMaxAttempts(OBJECT_STORE_MAX_ATTEMPTS_CONFIG) - .setTotalTimeoutDuration(OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES) - .setMaxRetryDelayDuration(OBJECT_STORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS) - .setInitialRetryDelayDuration(OBJECT_STORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS) - .setRetryDelayMultiplier(OBJECT_STORE_RETRY_DELAY_MULTIPLIER_CONFIG) + .setMaxAttempts(ObjectStoreConstants.OBJECT_STORE_MAX_ATTEMPTS_CONFIG) + .setTotalTimeoutDuration(ObjectStoreConstants.OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES) + .setMaxRetryDelayDuration(ObjectStoreConstants.OBJECT_STORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS) + .setInitialRetryDelayDuration( + ObjectStoreConstants.OBJECT_STORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS) + .setRetryDelayMultiplier(ObjectStoreConstants.OBJECT_STORE_RETRY_DELAY_MULTIPLIER_CONFIG) .build()) .build() .getService(); diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreConstants.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreConstants.java new file mode 100644 index 0000000000..ba1e651266 --- /dev/null +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreConstants.java @@ -0,0 +1,15 @@ +package org.cloudfoundry.multiapps.controller.persistence.util; + +import java.time.Duration; + +public class ObjectStoreConstants { + + private ObjectStoreConstants() { + } + + public static final int OBJECT_STORE_MAX_ATTEMPTS_CONFIG = 6; + public static final double OBJECT_STORE_RETRY_DELAY_MULTIPLIER_CONFIG = 2.0; + public static final Duration OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES = Duration.ofMinutes(10); + public static final Duration OBJECT_STORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS = Duration.ofSeconds(10); + public static final Duration OBJECT_STORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS = Duration.ofMillis(250); +} diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java new file mode 100644 index 0000000000..47aab944c2 --- /dev/null +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java @@ -0,0 +1,300 @@ +package org.cloudfoundry.multiapps.controller.persistence.services; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigInteger; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Stream; + +import com.azure.core.http.rest.PagedIterable; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.models.BlobItem; +import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.specialized.BlobInputStream; +import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class AzureObjectStoreFileStorageTest { + + @Mock + private BlobContainerClient blobContainerClient; + + @Mock + private BlobClient blobClient; + + @Mock + private PagedIterable pagedIterable; + + @Mock + private FileContentProcessor fileContentProcessor; + + @Mock + private BlobInputStream blobInputStream; + + private AzureObjectStoreFileStorage fileStorage; + private InputStream inputStream = new ByteArrayInputStream(new byte[] {}); + private final String TEST_SPACE_ID = UUID.randomUUID() + .toString(); + private final String TEST_SPACE_ID_2 = UUID.randomUUID() + .toString(); + private final String TEST_ID = UUID.randomUUID() + .toString(); + private final String TEST_ID_2 = UUID.randomUUID() + .toString(); + + private final String NAMESPACE = "namespace"; + private final String NAMESPACE_2 = "namespace_2"; + + @BeforeEach + void setUp() throws Exception { + MockitoAnnotations.openMocks(this) + .close(); + + fileStorage = new AzureObjectStoreFileStorage(Map.of()) { + + @Override + protected BlobContainerClient createContainerClient(Map credentials) { + return blobContainerClient; + } + }; + + when(blobContainerClient.getBlobClient(anyString())).thenReturn(blobClient); + } + + @Test + void testAddFileWithSuccessfulUpload() throws FileStorageException { + when(blobClient.uploadWithResponse(any(), any(), any())).thenReturn(null); + FileEntry fileEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + + fileStorage.addFile(fileEntry, inputStream); + + verify(blobClient).uploadWithResponse(any(), any(), any()); + } + + @Test + void testAddFileWithFailedUpload() { + doThrow(BlobStorageException.class).when(blobClient) + .uploadWithResponse(any(), any(), any()); + FileEntry fileEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + + assertThrows(FileStorageException.class, () -> fileStorage.addFile(fileEntry, inputStream)); + } + + @Test + void testGetFileEntriesWithoutContentWithoutMatches() throws FileStorageException { + setupDeleteMethods(createBlobItem(TEST_ID, TEST_SPACE_ID, NAMESPACE, LocalDateTime.now()), + createBlobItem(TEST_ID_2, TEST_SPACE_ID_2, NAMESPACE, LocalDateTime.now())); + FileEntry fileEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + + List fileEntries = fileStorage.getFileEntriesWithoutContent(List.of(fileEntry)); + + assertEquals(0, fileEntries.size()); + verify(blobContainerClient).listBlobs(); + } + + @Test + void testGetFileEntriesWithoutContent() throws FileStorageException { + setupDeleteMethods(createSecondTestBlobItem()); + + FileEntry fileEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + List fileEntries = fileStorage.getFileEntriesWithoutContent(List.of(fileEntry)); + + assertEquals(TEST_ID, fileEntries.get(0) + .getId()); + assertEquals(TEST_SPACE_ID, fileEntries.get(0) + .getSpace()); + assertEquals(1, fileEntries.size()); + verify(blobContainerClient).listBlobs(); + } + + @Test + void testDeleteFile() throws FileStorageException { + when(blobClient.deleteIfExists()).thenReturn(false); + + fileStorage.deleteFile(TEST_ID, TEST_SPACE_ID); + verify(blobClient).deleteIfExists(); + } + + @Test + void testTestConnection() { + fileStorage.testConnection(); + verify(blobContainerClient).getBlobClient("test"); + } + + @Test + void testGetContainerUriEndpointWithEmptyCredentials() { + assertNull(fileStorage.getContainerUriEndpoint(Map.of())); + } + + @Test + void testGetContainerUriEndpointWithInvalidContainerUri() { + assertThrows(IllegalStateException.class, () -> fileStorage.getContainerUriEndpoint(Map.of("container_uri", ""))); + } + + @Test + void testGetContainerUriEndpointWithValidContainerUri() { + assertEquals("https://google.com", fileStorage.getContainerUriEndpoint(Map.of("container_uri", "https://google.com"))); + } + + @Test + void testDeleteFileWithException() { + doThrow(new BlobStorageException("", null, null)).when(blobClient) + .deleteIfExists(); + + assertThrows(FileStorageException.class, () -> fileStorage.deleteFile(TEST_ID, TEST_SPACE_ID)); + verify(blobClient).deleteIfExists(); + } + + @Test + void testProcessFileContent() throws FileStorageException, IOException { + when(blobClient.openInputStream()).thenReturn(blobInputStream); + fileStorage.processFileContent(TEST_SPACE_ID, TEST_ID, fileContentProcessor); + + verify(fileContentProcessor).process(blobInputStream); + } + + @Test + void testProcessFileContentWithException() { + doThrow(new BlobStorageException("", null, null)).when(blobClient) + .openInputStream(); + + assertThrows(FileStorageException.class, () -> fileStorage.processFileContent(TEST_SPACE_ID, TEST_ID, fileContentProcessor)); + } + + @Test + void testDeleteFilesBySpaceIdsWithAllMatchingItems() throws FileStorageException { + setupDeleteMethods(createFirstTestBlobItem(), createSecondTestBlobItem()); + + fileStorage.deleteFilesBySpaceIds(List.of(TEST_SPACE_ID, TEST_SPACE_ID_2)); + + verify(blobClient, times(2)).deleteIfExists(); + } + + @Test + void testDeleteFilesBySpaceIdsWithOneMatchingItem() throws FileStorageException { + setupDeleteMethods(createFirstTestBlobItem(), createSecondTestBlobItem()); + + fileStorage.deleteFilesBySpaceIds(List.of(TEST_SPACE_ID)); + + verify(blobClient).deleteIfExists(); + } + + @Test + void testDeleteFilesBySpaceIdsWithoutMatchingItem() throws FileStorageException { + setupDeleteMethods(); + + fileStorage.deleteFilesBySpaceIds(List.of(TEST_SPACE_ID, TEST_SPACE_ID_2)); + + verify(blobClient, times(0)).deleteIfExists(); + } + + @Test + void testDeleteFilesBySpaceAndNamespaceWithOneMatch() { + setupDeleteMethods(createFirstTestBlobItem(), createSecondTestBlobItem()); + when(blobContainerClient.listBlobs()).thenReturn(pagedIterable); + when(blobClient.deleteIfExists()).thenReturn(true); + + fileStorage.deleteFilesBySpaceAndNamespace(TEST_SPACE_ID, NAMESPACE); + + verify(blobClient, times(1)).deleteIfExists(); + } + + @Test + void testDeleteFilesModifiedBefore() throws FileStorageException { + long currentMillis = System.currentTimeMillis(); + long oldFilesTtl = 1000 * 60 * 10; // 10min + setupDeleteMethods(createFirstTestBlobItem(), createSecondTestBlobItem()); + + int deletedFiles = fileStorage.deleteFilesModifiedBefore(LocalDateTime.ofInstant(Instant.ofEpochMilli(currentMillis - oldFilesTtl), + ZoneId.systemDefault())); + + assertEquals(2, deletedFiles); + } + + @Test + void testOpenInputStream() throws FileStorageException { + when(blobClient.openInputStream()).thenReturn(null); + + fileStorage.openInputStream(TEST_SPACE_ID_2, TEST_ID); + + verify(blobContainerClient).getBlobClient(TEST_ID); + verify(blobClient).openInputStream(); + } + + @Test + void testOpenInputStreamWithException() { + doThrow(new BlobStorageException(null, null, null)).when(blobClient) + .openInputStream(); + assertThrows(FileStorageException.class, () -> fileStorage.openInputStream(TEST_SPACE_ID_2, TEST_ID)); + + verify(blobContainerClient).getBlobClient(TEST_ID); + verify(blobClient).openInputStream(); + } + + @Test + void testDeleteFilesByIds() throws FileStorageException { + setupDeleteMethods(createFirstTestBlobItem(), createSecondTestBlobItem()); + + fileStorage.deleteFilesByIds(List.of(TEST_ID)); + + verify(blobClient).deleteIfExists(); + } + + private void setupDeleteMethods(BlobItem... blobItems) { + when(pagedIterable.stream()).thenReturn(Stream.of(blobItems)); + when(blobContainerClient.listBlobs(any(), any())).thenReturn(pagedIterable); + when(blobContainerClient.listBlobs()).thenReturn(pagedIterable); + when(blobClient.deleteIfExists()).thenReturn(true); + } + + public static FileEntry createFileEntry(String space, String id) { + return ImmutableFileEntry.builder() + .space(space) + .size(BigInteger.TEN) + .modified(LocalDateTime.now()) + .id(id) + .build(); + } + + private BlobItem createFirstTestBlobItem() { + long currentMillis = System.currentTimeMillis(); + long pastMoment = currentMillis - 1000 * 60 * 15; // before 15min + return createBlobItem(TEST_ID, TEST_SPACE_ID, NAMESPACE, + LocalDateTime.ofInstant(Instant.ofEpochMilli(pastMoment), ZoneId.systemDefault())); + } + + private BlobItem createSecondTestBlobItem() { + long currentMillis = System.currentTimeMillis(); + long pastMoment = currentMillis - 1000 * 60 * 15; // before 15min + return createBlobItem(TEST_ID_2, TEST_SPACE_ID_2, NAMESPACE_2, + LocalDateTime.ofInstant(Instant.ofEpochMilli(pastMoment), ZoneId.systemDefault())); + } + + private BlobItem createBlobItem(String name, String spaceId, String namespace, LocalDateTime modificationTime) { + BlobItem blobItem = new BlobItem(); + blobItem.setName(name); + blobItem.setMetadata(Map.of("space", spaceId, "namespace", namespace, "modified", modificationTime.toString())); + return blobItem; + } +} diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java index 0001e13be1..bf6adecbed 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java @@ -16,6 +16,7 @@ import org.apache.commons.lang3.StringUtils; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.core.util.UriUtil; +import org.cloudfoundry.multiapps.controller.persistence.services.AzureObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.GcpObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.JCloudsObjectStoreFileStorage; @@ -85,26 +86,17 @@ public FileStorage createObjectStoreFromFirstReachableProvider(Map gcpObjectStoreOpt = tryToCreateGcpObjectStore(exceptions); - if (gcpObjectStoreOpt.isPresent()) { - return gcpObjectStoreOpt.get(); - } throw buildNoValidObjectStoreException(exceptions); } private Optional createObjectStoreBasedOnProvider(String objectStoreProviderName, List providersServiceInfo, Map exceptions) { - Optional objectStoreServiceInfoOptional = getAppropriateProvider(objectStoreProviderName, - providersServiceInfo); - Optional createdObjectStore; - if (objectStoreServiceInfoOptional.isPresent()) { - ObjectStoreServiceInfo objectStoreServiceInfo = objectStoreServiceInfoOptional.get(); - createdObjectStore = tryToCreateObjectStore(objectStoreServiceInfo, exceptions); - } else { - createdObjectStore = tryToCreateGcpObjectStore(exceptions); - } - return createdObjectStore; + return switch (objectStoreProviderName) { + case Constants.AZURE -> tryToCreateSdkObjectStore(exceptions, Constants.AZUREBLOB); + case Constants.GCP -> tryToCreateSdkObjectStore(exceptions, Constants.GOOGLE_CLOUD_STORAGE); + default -> tryToCreateJCloudsObjectStore(objectStoreProviderName, providersServiceInfo, exceptions); + }; } private Optional getAppropriateProvider(String objectStoreProviderName, @@ -115,9 +107,21 @@ private Optional getAppropriateProvider(String objectSto .findFirst(); } - private Optional tryToCreateGcpObjectStore(Map exceptions) { + private Optional tryToCreateJCloudsObjectStore(String objectStoreProviderName, + List providersServiceInfo, + Map exceptions) { + Optional objectStoreServiceInfoOptional = getAppropriateProvider(objectStoreProviderName, + providersServiceInfo); + if (objectStoreServiceInfoOptional.isPresent()) { + ObjectStoreServiceInfo objectStoreServiceInfo = objectStoreServiceInfoOptional.get(); + return tryToCreateObjectStore(objectStoreServiceInfo, exceptions); + } + return Optional.empty(); + } + + private Optional tryToCreateSdkObjectStore(Map exceptions, String providerName) { return tryToCreateObjectStore(ImmutableObjectStoreServiceInfo.builder() - .provider(Constants.GOOGLE_CLOUD_STORAGE) + .provider(providerName) .build(), exceptions); } @@ -135,12 +139,14 @@ private Optional tryToCreateObjectStore(ObjectStoreServiceInfo obje } private FileStorage getFileStorageBasedOnProvider(ObjectStoreServiceInfo objectStoreServiceInfo) { - if (Constants.GOOGLE_CLOUD_STORAGE.equals(objectStoreServiceInfo.getProvider())) { - return createGcpFileStorage(); - } else { - BlobStoreContext context = getBlobStoreContext(objectStoreServiceInfo); - return createFileStorage(objectStoreServiceInfo, context); - } + return switch (objectStoreServiceInfo.getProvider()) { + case Constants.GOOGLE_CLOUD_STORAGE -> createGcpFileStorage(objectStoreServiceInfo); + case Constants.AZUREBLOB -> createAzureFileStorage(objectStoreServiceInfo); + default -> { + BlobStoreContext context = getBlobStoreContext(objectStoreServiceInfo); + yield createFileStorage(objectStoreServiceInfo, context); + } + }; } private boolean isObjectStoreEnvValid(String objectStoreProviderName) { @@ -226,9 +232,12 @@ protected JCloudsObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo return new JCloudsObjectStoreFileStorage(context.getBlobStore(), objectStoreServiceInfo.getContainer()); } - protected GcpObjectStoreFileStorage createGcpFileStorage() { - Map credentials = getServiceCredentials(); - return new GcpObjectStoreFileStorage(credentials); + protected GcpObjectStoreFileStorage createGcpFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { + return new GcpObjectStoreFileStorage(objectStoreServiceInfo.getCredentials()); + } + + protected AzureObjectStoreFileStorage createAzureFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { + return new AzureObjectStoreFileStorage(objectStoreServiceInfo.getCredentials()); } @Override diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java index 86c445e306..a822250a15 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java @@ -1,5 +1,7 @@ package org.cloudfoundry.multiapps.controller.web.configuration.service; +import java.util.Map; + import com.google.common.base.Supplier; import org.cloudfoundry.multiapps.common.Nullable; import org.immutables.value.Value; @@ -30,4 +32,7 @@ public interface ObjectStoreServiceInfo { @Nullable String getHost(); + + @Nullable + Map getCredentials(); } diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java index 57fd802b88..f215a78f68 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java @@ -1,18 +1,15 @@ package org.cloudfoundry.multiapps.controller.web.configuration.service; -import java.net.MalformedURLException; -import java.net.URL; import java.util.List; import java.util.Map; import org.cloudfoundry.multiapps.controller.web.Constants; -import org.cloudfoundry.multiapps.controller.web.Messages; public class ObjectStoreServiceInfoCreator { public List getAllProvidersServiceInfo(Map credentials) { return List.of(createServiceInfoForAws(credentials), createServiceInfoForAliCloud(credentials), - createServiceInfoForAzure(credentials), createServiceInfoForCcee(credentials)); + createServiceInfoForCcee(credentials), createServiceInfoForAzure(credentials), createServiceInfoForGcp(credentials)); } private ObjectStoreServiceInfo createServiceInfoForAws(Map credentials) { @@ -45,20 +42,6 @@ private ObjectStoreServiceInfo createServiceInfoForAliCloud(Map .build(); } - private ObjectStoreServiceInfo createServiceInfoForAzure(Map credentials) { - String accountName = (String) credentials.get(Constants.ACCOUNT_NAME); - String sasToken = (String) credentials.get(Constants.SAS_TOKEN); - String containerName = (String) credentials.get(Constants.CONTAINER_NAME); - URL containerUrl = getContainerUriEndpoint(credentials); - return ImmutableObjectStoreServiceInfo.builder() - .provider(Constants.AZUREBLOB) - .identity(accountName) - .credential(sasToken) - .endpoint(containerUrl == null ? null : containerUrl.toString()) - .container(containerName) - .build(); - } - private ObjectStoreServiceInfo createServiceInfoForCcee(Map credentials) { String accessKeyId = (String) credentials.get(Constants.ACCESS_KEY_ID); String containerName = (String) credentials.get(Constants.CONTAINER_NAME_WITH_DASH); @@ -75,15 +58,17 @@ private ObjectStoreServiceInfo createServiceInfoForCcee(Map cred .build(); } - private URL getContainerUriEndpoint(Map credentials) { - if (!credentials.containsKey(Constants.CONTAINER_URI)) { - return null; - } - try { - URL containerUri = new URL((String) credentials.get(Constants.CONTAINER_URI)); - return new URL(containerUri.getProtocol(), containerUri.getHost(), containerUri.getPort(), ""); - } catch (MalformedURLException e) { - throw new IllegalStateException(Messages.CANNOT_PARSE_CONTAINER_URI_OF_OBJECT_STORE, e); - } + private ObjectStoreServiceInfo createServiceInfoForAzure(Map credentials) { + return ImmutableObjectStoreServiceInfo.builder() + .provider(Constants.AZUREBLOB) + .credentials(credentials) + .build(); + } + + private ObjectStoreServiceInfo createServiceInfoForGcp(Map credentials) { + return ImmutableObjectStoreServiceInfo.builder() + .provider(Constants.GOOGLE_CLOUD_STORAGE) + .credentials(credentials) + .build(); } } diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java index 5b5617b7a6..dc5fc6bded 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java @@ -8,6 +8,7 @@ import io.pivotal.cfenv.core.CfCredentials; import io.pivotal.cfenv.core.CfService; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; +import org.cloudfoundry.multiapps.controller.persistence.services.AzureObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.GcpObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.JCloudsObjectStoreFileStorage; @@ -27,6 +28,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.Mockito.doThrow; @@ -54,6 +56,9 @@ class ObjectStoreFileStorageFactoryBeanTest { @Mock private GcpObjectStoreFileStorage gcpObjectStoreFileStorage; + @Mock + private AzureObjectStoreFileStorage azureObjectStoreFileStorage; + @BeforeEach void setUp() throws Exception { MockitoAnnotations.openMocks(this) @@ -79,7 +84,7 @@ void testObjectStoreCreationWithValidServiceInstance() { } @Test - void testObjectStoreCreationWhenEnvIsValid() { + void testObjectStoreCreationWhenEnvIsValidForAws() { mockCfService(); when(applicationConfiguration.getObjectStoreClientType()).thenReturn(Constants.AWS); ObjectStoreFileStorageFactoryBean spy = spy(objectStoreFileStorageFactoryBean); @@ -92,6 +97,41 @@ void testObjectStoreCreationWhenEnvIsValid() { .createObjectStoreFromFirstReachableProvider(anyMap(), anyList()); verify(jCloudsObjectStoreFileStorage, times(1)) .testConnection(); + assertTrue(createdObjectStoreFileStorage instanceof JCloudsObjectStoreFileStorage); + } + + @Test + void testObjectStoreCreationWhenEnvIsValidForAzure() { + mockCfService(); + when(applicationConfiguration.getObjectStoreClientType()).thenReturn(Constants.AZURE); + ObjectStoreFileStorageFactoryBean spy = spy(objectStoreFileStorageFactoryBean); + + spy.afterPropertiesSet(); + FileStorage createdObjectStoreFileStorage = spy.getObject(); + + assertNotNull(createdObjectStoreFileStorage); + verify(spy, never()) + .createObjectStoreFromFirstReachableProvider(anyMap(), anyList()); + verify(azureObjectStoreFileStorage, times(1)) + .testConnection(); + assertTrue(createdObjectStoreFileStorage instanceof AzureObjectStoreFileStorage); + } + + @Test + void testObjectStoreCreationWhenEnvIsValid() { + mockCfService(); + when(applicationConfiguration.getObjectStoreClientType()).thenReturn(Constants.GCP); + ObjectStoreFileStorageFactoryBean spy = spy(objectStoreFileStorageFactoryBean); + + spy.afterPropertiesSet(); + FileStorage createdObjectStoreFileStorage = spy.getObject(); + + assertNotNull(createdObjectStoreFileStorage); + verify(spy, never()) + .createObjectStoreFromFirstReachableProvider(anyMap(), anyList()); + verify(gcpObjectStoreFileStorage, times(1)) + .testConnection(); + assertTrue(createdObjectStoreFileStorage instanceof GcpObjectStoreFileStorage); } @Test @@ -127,6 +167,8 @@ void testObjectStoreCreationWithoutValidServiceInstance() { .testConnection(); doThrow(new IllegalStateException("Cannot create object store")).when(gcpObjectStoreFileStorage) .testConnection(); + doThrow(new IllegalStateException("Cannot create object store")).when(azureObjectStoreFileStorage) + .testConnection(); Exception exception = assertThrows(IllegalStateException.class, () -> objectStoreFileStorageFactoryBean.afterPropertiesSet()); assertEquals(Messages.NO_VALID_OBJECT_STORE_CONFIGURATION_FOUND, exception.getMessage()); } @@ -160,10 +202,15 @@ protected JCloudsObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo } @Override - protected GcpObjectStoreFileStorage createGcpFileStorage() { + protected GcpObjectStoreFileStorage createGcpFileStorage(ObjectStoreServiceInfo credentials) { return ObjectStoreFileStorageFactoryBeanTest.this.gcpObjectStoreFileStorage; } + @Override + protected AzureObjectStoreFileStorage createAzureFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { + return ObjectStoreFileStorageFactoryBeanTest.this.azureObjectStoreFileStorage; + } + @Override public List getProvidersServiceInfo() { CfService service = environmentServicesFinder.findService("deploy-service-os"); diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java index 6439250543..bc95563fb6 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java @@ -1,7 +1,5 @@ package org.cloudfoundry.multiapps.controller.web.configuration.service; -import java.net.MalformedURLException; -import java.net.URL; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -26,10 +24,7 @@ class ObjectStoreServiceInfoCreatorTest { private static final String BUCKET_VALUE = "bucket_value"; private static final String REGION_VALUE = "region_value"; private static final String ENDPOINT_VALUE = "endpoint_value"; - private static final String ACCOUNT_NAME_VALUE = "account_name_value"; - private static final String SAS_TOKEN_VALUE = "sas_token_value"; - private static final String CONTAINER_NAME_VALUE = "container_name_value"; - private static final String CONTAINER_URI_VALUE = "https://container.com:8080"; + private static final Map CREDENTIALS = Map.of("test", "test1"); private ObjectStoreServiceInfoCreator objectStoreServiceInfoCreator; @@ -38,10 +33,11 @@ void setUp() { objectStoreServiceInfoCreator = new ObjectStoreServiceInfoCreatorMock(); } - static Stream testDifferentProviders() throws MalformedURLException { + static Stream testDifferentProviders() { return Stream.of(Arguments.of(buildCfService(buildAliCloudCredentials()), buildAliCloudObjectStoreServiceInfo()), Arguments.of(buildCfService(buildAwsCredentials()), buildAwsObjectStoreServiceInfo()), - Arguments.of(buildCfService(buildAzureCredentials()), buildAzureObjectStoreServiceInfo())); + Arguments.of(buildCfService(buildSdkCredentials()), buildAzureObjectStoreServiceInfo()), + Arguments.of(buildCfService(buildSdkCredentials()), buildGcoObjectStoreServiceInfo())); } @ParameterizedTest @@ -99,22 +95,21 @@ private static ObjectStoreServiceInfo buildAwsObjectStoreServiceInfo() { .build(); } - private static Map buildAzureCredentials() { - Map credentials = new HashMap<>(); - credentials.put(Constants.ACCOUNT_NAME, ACCOUNT_NAME_VALUE); - credentials.put(Constants.SAS_TOKEN, SAS_TOKEN_VALUE); - credentials.put(Constants.CONTAINER_NAME, CONTAINER_NAME_VALUE); - credentials.put(Constants.CONTAINER_URI, CONTAINER_URI_VALUE); - return credentials; + private static Map buildSdkCredentials() { + return CREDENTIALS; } - private static ObjectStoreServiceInfo buildAzureObjectStoreServiceInfo() throws MalformedURLException { + private static ObjectStoreServiceInfo buildAzureObjectStoreServiceInfo() { return ImmutableObjectStoreServiceInfo.builder() .provider(Constants.AZUREBLOB) - .identity(ACCOUNT_NAME_VALUE) - .credential(SAS_TOKEN_VALUE) - .endpoint(new URL("https", "container.com", 8080, "").toString()) - .container(CONTAINER_NAME_VALUE) + .credentials(CREDENTIALS) + .build(); + } + + private static ObjectStoreServiceInfo buildGcoObjectStoreServiceInfo() { + return ImmutableObjectStoreServiceInfo.builder() + .provider(Constants.GOOGLE_CLOUD_STORAGE) + .credentials(CREDENTIALS) .build(); } diff --git a/pom.xml b/pom.xml index d83ebfc187..2e689ab3a5 100644 --- a/pom.xml +++ b/pom.xml @@ -60,6 +60,8 @@ 1.3.5 2.62.1 0.128.11 + 12.33.1 + 1.13.3 multiapps-controller-client @@ -302,6 +304,16 @@ ${google-cloud-nio.version} test + + com.azure + azure-storage-blob + ${azure-storage-blob.version} + + + com.azure + azure-core-http-okhttp + ${azure-core-http-okhttp.version} + org.immutables