From 9edab1215b5c8dc5cf65e54933715d5bf2d7a3d0 Mon Sep 17 00:00:00 2001 From: lichen782 Date: Mon, 2 Sep 2019 15:20:49 +0800 Subject: [PATCH] Support OssStorage Resource as WritableResource and add related unittest cases --- .../alibaba/cloud/examples/OssController.java | 36 ++- spring-cloud-alicloud-oss/pom.xml | 6 + .../oss/resource/OssStorageResource.java | 104 +++++++- .../alicloud/oss/resource/DummyOssClient.java | 80 ++++++ .../oss/resource/OssStorageResourceTest.java | 227 ++++++++++++++++++ 5 files changed, 434 insertions(+), 19 deletions(-) create mode 100644 spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/DummyOssClient.java create mode 100644 spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/OssStorageResourceTest.java diff --git a/spring-cloud-alibaba-examples/oss-example/src/main/java/com/alibaba/cloud/examples/OssController.java b/spring-cloud-alibaba-examples/oss-example/src/main/java/com/alibaba/cloud/examples/OssController.java index c1c9d261..30d51c72 100644 --- a/spring-cloud-alibaba-examples/oss-example/src/main/java/com/alibaba/cloud/examples/OssController.java +++ b/spring-cloud-alibaba-examples/oss-example/src/main/java/com/alibaba/cloud/examples/OssController.java @@ -1,7 +1,9 @@ package com.alibaba.cloud.examples; -import java.nio.charset.Charset; - +import com.alibaba.alicloud.oss.resource.OssStorageResource; +import com.aliyun.oss.OSS; +import com.aliyun.oss.common.utils.IOUtils; +import com.aliyun.oss.model.OSSObject; import org.apache.commons.codec.CharEncoding; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -10,9 +12,9 @@ import org.springframework.util.StreamUtils; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RestController; -import com.aliyun.oss.OSS; -import com.aliyun.oss.common.utils.IOUtils; -import com.aliyun.oss.model.OSSObject; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.Charset; /** * OSS Controller @@ -25,8 +27,11 @@ public class OssController { @Autowired private OSS ossClient; + @Value("classpath:/oss-test.json") + private Resource localFile; + @Value("oss://" + OssApplication.BUCKET_NAME + "/oss-test.json") - private Resource file; + private Resource remoteFile; @GetMapping("/upload") public String upload() { @@ -45,7 +50,7 @@ public class OssController { public String fileResource() { try { return "get file resource success. content: " + StreamUtils.copyToString( - file.getInputStream(), Charset.forName(CharEncoding.UTF_8)); + remoteFile.getInputStream(), Charset.forName(CharEncoding.UTF_8)); } catch (Exception e) { e.printStackTrace(); @@ -67,4 +72,21 @@ public class OssController { } } + @GetMapping("/upload2") + public String uploadWithOutputStream() { + OssStorageResource ossStorageResource = new OssStorageResource(this.ossClient, + "oss://" + OssApplication.BUCKET_NAME + "/oss-test.json", true); + try { + InputStream inputStream = localFile.getInputStream(); + OutputStream outputStream = ossStorageResource.getOutputStream(); + StreamUtils.copy(inputStream, outputStream); + inputStream.close(); + outputStream.close(); + } catch (Exception ex) { + ex.printStackTrace(); + return "upload with outputStream failed"; + } + return "upload success"; + } + } diff --git a/spring-cloud-alicloud-oss/pom.xml b/spring-cloud-alicloud-oss/pom.xml index 85318849..30664e8a 100644 --- a/spring-cloud-alicloud-oss/pom.xml +++ b/spring-cloud-alicloud-oss/pom.xml @@ -58,6 +58,12 @@ test + + org.mockito + mockito-core + test + + diff --git a/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageResource.java b/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageResource.java index 70825bae..957411c8 100644 --- a/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageResource.java +++ b/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageResource.java @@ -16,22 +16,25 @@ package com.alibaba.alicloud.oss.resource; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; - -import org.springframework.core.io.Resource; -import org.springframework.util.Assert; - import com.aliyun.oss.ClientException; import com.aliyun.oss.OSS; import com.aliyun.oss.OSSException; import com.aliyun.oss.model.Bucket; import com.aliyun.oss.model.OSSObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.io.Resource; +import org.springframework.core.io.WritableResource; +import org.springframework.util.Assert; + +import java.io.*; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** * Implements {@link Resource} for reading and writing objects in Aliyun Object Storage @@ -43,18 +46,30 @@ import com.aliyun.oss.model.OSSObject; * @see Bucket * @see OSSObject */ -public class OssStorageResource implements Resource { +public class OssStorageResource implements WritableResource { + + private static final Logger logger = LoggerFactory.getLogger(OssStorageResource.class); + + private static final String MESSAGE_KEY_NOT_EXIST = "The specified key does not exist."; private final OSS oss; private final String bucketName; private final String objectKey; private final URI location; + private final boolean autoCreateFiles; + + private final ExecutorService executorService; public OssStorageResource(OSS oss, String location) { + this(oss, location, false); + } + + public OssStorageResource(OSS oss, String location, boolean autoCreateFiles) { Assert.notNull(oss, "Object Storage Service can not be null"); Assert.isTrue(location.startsWith(OssStorageProtocolResolver.PROTOCOL), "Location must start with " + OssStorageProtocolResolver.PROTOCOL); this.oss = oss; + this.autoCreateFiles = autoCreateFiles; try { URI locationUri = new URI(location); this.bucketName = locationUri.getAuthority(); @@ -70,6 +85,14 @@ public class OssStorageResource implements Resource { catch (URISyntaxException e) { throw new IllegalArgumentException("Invalid location: " + location, e); } + + this.executorService = new ThreadPoolExecutor( + 1, 1, 60, TimeUnit.SECONDS, + new SynchronousQueue<>()); + } + + public boolean isAutoCreateFiles() { + return this.autoCreateFiles; } @Override @@ -193,4 +216,61 @@ public class OssStorageResource implements Resource { } } + public Bucket createBucket() { + return this.oss.createBucket(this.bucketName); + } + + @Override + public boolean isWritable() { + return !isBucket() && (this.autoCreateFiles || exists()); + } + + /** + * 获取一个OutputStream用于写操作。 + * 注意：写完成后必须关闭该流 + * @return + * @throws IOException + */ + @Override + public OutputStream getOutputStream() throws IOException { + if (isBucket()) { + throw new IllegalStateException( + "Cannot open an output stream to a bucket: '" + getURI() + "'"); + } + else { + OSSObject ossObject; + + try { + ossObject = this.getOSSObject(); + } catch (OSSException ex) { + if (ex.getMessage() != null && ex.getMessage().startsWith(MESSAGE_KEY_NOT_EXIST)) { + ossObject = null; + } else { + throw ex; + } + } + + if (ossObject == null ) { + if (!this.autoCreateFiles) { + throw new FileNotFoundException("The object was not found: " + getURI()); + } + + } + + PipedInputStream in = new PipedInputStream(); + final PipedOutputStream out = new PipedOutputStream(in); + + executorService.submit(() -> { + try { + OssStorageResource.this.oss.putObject(bucketName, objectKey, in); + } catch (Exception ex) { + logger.error("Failed to put object", ex); + } + }); + + return out; + } + + } + } diff --git a/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/DummyOssClient.java b/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/DummyOssClient.java new file mode 100644 index 00000000..54b52278 --- /dev/null +++ b/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/DummyOssClient.java @@ -0,0 +1,80 @@ +package com.alibaba.alicloud.oss.resource; + +import com.aliyun.oss.model.Bucket; +import com.aliyun.oss.model.OSSObject; +import com.aliyun.oss.model.ObjectMetadata; +import com.aliyun.oss.model.PutObjectResult; +import org.springframework.util.StreamUtils; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +/** + * @author lich + * @date 2019/8/30 + */ +public class DummyOssClient { + + private Map storeMap = new ConcurrentHashMap<>(); + + private Map bucketSet = new HashMap<>(); + + public String getStoreKey(String bucketName, String objectKey) { + return String.join(".", bucketName, objectKey); + } + + public PutObjectResult putObject(String bucketName, String objectKey, InputStream inputStream) { + + try { + byte[] result = StreamUtils.copyToByteArray(inputStream); + storeMap.put(getStoreKey(bucketName, objectKey), result); + } catch (IOException ex) { + throw new RuntimeException(ex); + } finally { + try { + inputStream.close(); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + } + + + return new PutObjectResult(); + } + + public OSSObject getOSSObject(String bucketName, String objectKey) { + byte[] value = storeMap.get(this.getStoreKey(bucketName, objectKey)); + if (value == null) { + return null; + } + OSSObject ossObject = new OSSObject(); + ossObject.setBucketName(bucketName); + ossObject.setKey(objectKey); + InputStream inputStream = new ByteArrayInputStream(value); + ossObject.setObjectContent(inputStream); + + ObjectMetadata objectMetadata = new ObjectMetadata(); + objectMetadata.setContentLength(value.length); + ossObject.setObjectMetadata(objectMetadata); + + return ossObject; + } + + public Bucket createBucket(String bucketName) { + if (bucketSet.containsKey(bucketName)) { + return bucketSet.get(bucketName); + } + Bucket bucket = new Bucket(); + bucket.setCreationDate(new Date()); + bucket.setName(bucketName); + bucketSet.put(bucketName, bucket); + return bucket; + } + + public List bucketList() { + return new ArrayList<>(bucketSet.values()); + } +} diff --git a/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/OssStorageResourceTest.java b/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/OssStorageResourceTest.java new file mode 100644 index 00000000..9949636c --- /dev/null +++ b/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/OssStorageResourceTest.java @@ -0,0 +1,227 @@ +package com.alibaba.alicloud.oss.resource; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.common.utils.IOUtils; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.core.io.Resource; +import org.springframework.core.io.WritableResource; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.util.StreamUtils; + +import java.io.*; +import java.nio.charset.Charset; +import java.util.Random; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; + +/** + * @author lich + * @date 2019/8/29 + */ + +@SpringBootTest +@RunWith(SpringRunner.class) +public class OssStorageResourceTest { + + /** + * Used to test exception messages and types. + */ + @Rule + public ExpectedException expectedEx = ExpectedException.none(); + + @Autowired + private OSS oss; + + @Value("oss://aliyun-test-bucket/") + private Resource bucketResource; + + @Value("oss://aliyun-test-bucket/myfilekey") + private Resource remoteResource; + + public static byte[] generateRandomBytes(int blen) { + byte[] array = new byte[blen]; + new Random().nextBytes(array); + return array; + } + + @Test + public void testResourceType() { + assertEquals(OssStorageResource.class, remoteResource.getClass()); + OssStorageResource ossStorageResource = (OssStorageResource)remoteResource; + assertEquals("myfilekey", ossStorageResource.getFilename()); + assertFalse(ossStorageResource.isBucket()); + } + + @Test + public void testValidObject() throws Exception { + assertTrue(remoteResource.exists()); + OssStorageResource ossStorageResource = (OssStorageResource)remoteResource; + assertTrue(ossStorageResource.bucketExists()); + assertEquals(4096L, remoteResource.contentLength()); + assertEquals("oss://aliyun-test-bucket/myfilekey", remoteResource.getURI().toString()); + assertEquals("myfilekey", remoteResource.getFilename()); + } + + @Test + public void testBucketResource() throws Exception { + assertTrue(bucketResource.exists()); + assertTrue(((OssStorageResource)this.bucketResource).isBucket()); + assertTrue(((OssStorageResource)this.bucketResource).bucketExists()); + assertEquals("oss://aliyun-test-bucket/", bucketResource.getURI().toString()); + assertEquals("aliyun-test-bucket", this.bucketResource.getFilename()); + } + + @Test + public void testBucketNotEndingInSlash() { + assertTrue(new OssStorageResource(this.oss, "oss://aliyun-test-bucket").isBucket()); + } + + @Test + public void testSpecifyPathCorrect() { + OssStorageResource ossStorageResource = new OssStorageResource ( + this.oss, "oss://aliyun-test-bucket/myfilekey", false); + + assertTrue(ossStorageResource.exists()); + } + + @Test + public void testSpecifyBucketCorrect() { + OssStorageResource ossStorageResource = new OssStorageResource( + this.oss, "oss://aliyun-test-bucket", false); + + assertTrue(ossStorageResource.isBucket()); + assertEquals("aliyun-test-bucket", ossStorageResource.getBucket().getName()); + assertTrue(ossStorageResource.exists()); + } + + @Test + public void testBucketOutputStream() throws IOException { + this.expectedEx.expect(IllegalStateException.class); + this.expectedEx.expectMessage("Cannot open an output stream to a bucket: 'oss://aliyun-test-bucket/'"); + ((WritableResource) this.bucketResource).getOutputStream(); + } + + @Test + public void testBucketInputStream() throws IOException { + this.expectedEx.expect(IllegalStateException.class); + this.expectedEx.expectMessage("Cannot open an input stream to a bucket: 'oss://aliyun-test-bucket/'"); + this.bucketResource.getInputStream(); + } + + @Test + public void testBucketContentLength() throws IOException { + this.expectedEx.expect(FileNotFoundException.class); + this.expectedEx.expectMessage("OSSObject not existed."); + this.bucketResource.contentLength(); + } + + @Test + public void testBucketFile() throws IOException { + this.expectedEx.expect(UnsupportedOperationException.class); + this.expectedEx.expectMessage("oss://aliyun-test-bucket/ cannot be resolved to absolute file path"); + this.bucketResource.getFile(); + } + + @Test + public void testBucketLastModified() throws IOException { + this.expectedEx.expect(FileNotFoundException.class); + this.expectedEx.expectMessage("OSSObject not existed."); + this.bucketResource.lastModified(); + } + + @Test + public void testBucketResourceStatuses() { + assertFalse(this.bucketResource.isOpen()); + assertFalse(((WritableResource) this.bucketResource).isWritable()); + assertTrue(this.bucketResource.exists()); + } + + @Test + public void testWritable() throws Exception { + assertTrue(this.remoteResource instanceof WritableResource); + WritableResource writableResource = (WritableResource) this.remoteResource; + assertTrue(writableResource.isWritable()); + writableResource.getOutputStream(); + } + + @Test + public void testWritableOutputStream() throws Exception { + String location = "oss://aliyun-test-bucket/test"; + OssStorageResource resource = new OssStorageResource(this.oss, location, true); + OutputStream os = resource.getOutputStream(); + assertNotNull(os); + + byte[] randomBytes = generateRandomBytes(1203); + String expectedString = new String(randomBytes); + + os.write(randomBytes); + os.close(); + + InputStream in = resource.getInputStream(); + + byte[] result = StreamUtils.copyToByteArray(in); + String actualString = new String(result); + + assertEquals(expectedString, actualString); + } + + + + /** + * Configuration for the tests. + */ + @Configuration + @Import(OssStorageProtocolResolver.class) + static class TestConfiguration { + + @Bean + public static OSS mockOSS() { + DummyOssClient dummyOssStub = new DummyOssClient(); + OSS oss = mock(OSS.class); + + doAnswer(invocation -> + dummyOssStub.putObject( + invocation.getArgument(0), + invocation.getArgument(1), + invocation.getArgument(2) + )) + .when(oss).putObject(Mockito.anyString(), Mockito.anyString(), Mockito.any(InputStream.class)); + + doAnswer(invocation -> + dummyOssStub.getOSSObject( + invocation.getArgument(0), + invocation.getArgument(1) + )) + .when(oss).getObject(Mockito.anyString(), Mockito.anyString()); + + doAnswer(invocation -> dummyOssStub.bucketList()) + .when(oss).listBuckets(); + + doAnswer(invocation -> dummyOssStub.createBucket(invocation.getArgument(0))) + .when(oss).createBucket(Mockito.anyString()); + + // prepare object + dummyOssStub.createBucket("aliyun-test-bucket"); + + byte[] content = generateRandomBytes(4096); + ByteArrayInputStream inputStream = new ByteArrayInputStream(content); + dummyOssStub.putObject("aliyun-test-bucket", "myfilekey", inputStream); + + return oss; + } + + } + +}