From 9edab1215b5c8dc5cf65e54933715d5bf2d7a3d0 Mon Sep 17 00:00:00 2001 From: lichen782 Date: Mon, 2 Sep 2019 15:20:49 +0800 Subject: [PATCH 1/6] Support OssStorage Resource as WritableResource and add related unittest cases --- .../alibaba/cloud/examples/OssController.java | 36 ++- spring-cloud-alicloud-oss/pom.xml | 6 + .../oss/resource/OssStorageResource.java | 104 +++++++- .../alicloud/oss/resource/DummyOssClient.java | 80 ++++++ .../oss/resource/OssStorageResourceTest.java | 227 ++++++++++++++++++ 5 files changed, 434 insertions(+), 19 deletions(-) create mode 100644 spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/DummyOssClient.java create mode 100644 spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/OssStorageResourceTest.java diff --git a/spring-cloud-alibaba-examples/oss-example/src/main/java/com/alibaba/cloud/examples/OssController.java b/spring-cloud-alibaba-examples/oss-example/src/main/java/com/alibaba/cloud/examples/OssController.java index c1c9d261..30d51c72 100644 --- a/spring-cloud-alibaba-examples/oss-example/src/main/java/com/alibaba/cloud/examples/OssController.java +++ b/spring-cloud-alibaba-examples/oss-example/src/main/java/com/alibaba/cloud/examples/OssController.java @@ -1,7 +1,9 @@ package com.alibaba.cloud.examples; -import java.nio.charset.Charset; - +import com.alibaba.alicloud.oss.resource.OssStorageResource; +import com.aliyun.oss.OSS; +import com.aliyun.oss.common.utils.IOUtils; +import com.aliyun.oss.model.OSSObject; import org.apache.commons.codec.CharEncoding; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -10,9 +12,9 @@ import org.springframework.util.StreamUtils; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RestController; -import com.aliyun.oss.OSS; -import com.aliyun.oss.common.utils.IOUtils; -import com.aliyun.oss.model.OSSObject; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.Charset; /** * OSS Controller @@ -25,8 +27,11 @@ public class OssController { @Autowired private OSS ossClient; + @Value("classpath:/oss-test.json") + private Resource localFile; + @Value("oss://" + OssApplication.BUCKET_NAME + "/oss-test.json") - private Resource file; + private Resource remoteFile; @GetMapping("/upload") public String upload() { @@ -45,7 +50,7 @@ public class OssController { public String fileResource() { try { return "get file resource success. content: " + StreamUtils.copyToString( - file.getInputStream(), Charset.forName(CharEncoding.UTF_8)); + remoteFile.getInputStream(), Charset.forName(CharEncoding.UTF_8)); } catch (Exception e) { e.printStackTrace(); @@ -67,4 +72,21 @@ public class OssController { } } + @GetMapping("/upload2") + public String uploadWithOutputStream() { + OssStorageResource ossStorageResource = new OssStorageResource(this.ossClient, + "oss://" + OssApplication.BUCKET_NAME + "/oss-test.json", true); + try { + InputStream inputStream = localFile.getInputStream(); + OutputStream outputStream = ossStorageResource.getOutputStream(); + StreamUtils.copy(inputStream, outputStream); + inputStream.close(); + outputStream.close(); + } catch (Exception ex) { + ex.printStackTrace(); + return "upload with outputStream failed"; + } + return "upload success"; + } + } diff --git a/spring-cloud-alicloud-oss/pom.xml b/spring-cloud-alicloud-oss/pom.xml index 85318849..30664e8a 100644 --- a/spring-cloud-alicloud-oss/pom.xml +++ b/spring-cloud-alicloud-oss/pom.xml @@ -58,6 +58,12 @@ test + + org.mockito + mockito-core + test + + diff --git a/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageResource.java b/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageResource.java index 70825bae..957411c8 100644 --- a/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageResource.java +++ b/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageResource.java @@ -16,22 +16,25 @@ package com.alibaba.alicloud.oss.resource; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; - -import org.springframework.core.io.Resource; -import org.springframework.util.Assert; - import com.aliyun.oss.ClientException; import com.aliyun.oss.OSS; import com.aliyun.oss.OSSException; import com.aliyun.oss.model.Bucket; import com.aliyun.oss.model.OSSObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.io.Resource; +import org.springframework.core.io.WritableResource; +import org.springframework.util.Assert; + +import java.io.*; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** * Implements {@link Resource} for reading and writing objects in Aliyun Object Storage @@ -43,18 +46,30 @@ import com.aliyun.oss.model.OSSObject; * @see Bucket * @see OSSObject */ -public class OssStorageResource implements Resource { +public class OssStorageResource implements WritableResource { + + private static final Logger logger = LoggerFactory.getLogger(OssStorageResource.class); + + private static final String MESSAGE_KEY_NOT_EXIST = "The specified key does not exist."; private final OSS oss; private final String bucketName; private final String objectKey; private final URI location; + private final boolean autoCreateFiles; + + private final ExecutorService executorService; public OssStorageResource(OSS oss, String location) { + this(oss, location, false); + } + + public OssStorageResource(OSS oss, String location, boolean autoCreateFiles) { Assert.notNull(oss, "Object Storage Service can not be null"); Assert.isTrue(location.startsWith(OssStorageProtocolResolver.PROTOCOL), "Location must start with " + OssStorageProtocolResolver.PROTOCOL); this.oss = oss; + this.autoCreateFiles = autoCreateFiles; try { URI locationUri = new URI(location); this.bucketName = locationUri.getAuthority(); @@ -70,6 +85,14 @@ public class OssStorageResource implements Resource { catch (URISyntaxException e) { throw new IllegalArgumentException("Invalid location: " + location, e); } + + this.executorService = new ThreadPoolExecutor( + 1, 1, 60, TimeUnit.SECONDS, + new SynchronousQueue<>()); + } + + public boolean isAutoCreateFiles() { + return this.autoCreateFiles; } @Override @@ -193,4 +216,61 @@ public class OssStorageResource implements Resource { } } + public Bucket createBucket() { + return this.oss.createBucket(this.bucketName); + } + + @Override + public boolean isWritable() { + return !isBucket() && (this.autoCreateFiles || exists()); + } + + /** + * 获取一个OutputStream用于写操作。 + * 注意：写完成后必须关闭该流 + * @return + * @throws IOException + */ + @Override + public OutputStream getOutputStream() throws IOException { + if (isBucket()) { + throw new IllegalStateException( + "Cannot open an output stream to a bucket: '" + getURI() + "'"); + } + else { + OSSObject ossObject; + + try { + ossObject = this.getOSSObject(); + } catch (OSSException ex) { + if (ex.getMessage() != null && ex.getMessage().startsWith(MESSAGE_KEY_NOT_EXIST)) { + ossObject = null; + } else { + throw ex; + } + } + + if (ossObject == null ) { + if (!this.autoCreateFiles) { + throw new FileNotFoundException("The object was not found: " + getURI()); + } + + } + + PipedInputStream in = new PipedInputStream(); + final PipedOutputStream out = new PipedOutputStream(in); + + executorService.submit(() -> { + try { + OssStorageResource.this.oss.putObject(bucketName, objectKey, in); + } catch (Exception ex) { + logger.error("Failed to put object", ex); + } + }); + + return out; + } + + } + } diff --git a/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/DummyOssClient.java b/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/DummyOssClient.java new file mode 100644 index 00000000..54b52278 --- /dev/null +++ b/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/DummyOssClient.java @@ -0,0 +1,80 @@ +package com.alibaba.alicloud.oss.resource; + +import com.aliyun.oss.model.Bucket; +import com.aliyun.oss.model.OSSObject; +import com.aliyun.oss.model.ObjectMetadata; +import com.aliyun.oss.model.PutObjectResult; +import org.springframework.util.StreamUtils; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +/** + * @author lich + * @date 2019/8/30 + */ +public class DummyOssClient { + + private Map storeMap = new ConcurrentHashMap<>(); + + private Map bucketSet = new HashMap<>(); + + public String getStoreKey(String bucketName, String objectKey) { + return String.join(".", bucketName, objectKey); + } + + public PutObjectResult putObject(String bucketName, String objectKey, InputStream inputStream) { + + try { + byte[] result = StreamUtils.copyToByteArray(inputStream); + storeMap.put(getStoreKey(bucketName, objectKey), result); + } catch (IOException ex) { + throw new RuntimeException(ex); + } finally { + try { + inputStream.close(); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + } + + + return new PutObjectResult(); + } + + public OSSObject getOSSObject(String bucketName, String objectKey) { + byte[] value = storeMap.get(this.getStoreKey(bucketName, objectKey)); + if (value == null) { + return null; + } + OSSObject ossObject = new OSSObject(); + ossObject.setBucketName(bucketName); + ossObject.setKey(objectKey); + InputStream inputStream = new ByteArrayInputStream(value); + ossObject.setObjectContent(inputStream); + + ObjectMetadata objectMetadata = new ObjectMetadata(); + objectMetadata.setContentLength(value.length); + ossObject.setObjectMetadata(objectMetadata); + + return ossObject; + } + + public Bucket createBucket(String bucketName) { + if (bucketSet.containsKey(bucketName)) { + return bucketSet.get(bucketName); + } + Bucket bucket = new Bucket(); + bucket.setCreationDate(new Date()); + bucket.setName(bucketName); + bucketSet.put(bucketName, bucket); + return bucket; + } + + public List bucketList() { + return new ArrayList<>(bucketSet.values()); + } +} diff --git a/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/OssStorageResourceTest.java b/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/OssStorageResourceTest.java new file mode 100644 index 00000000..9949636c --- /dev/null +++ b/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/OssStorageResourceTest.java @@ -0,0 +1,227 @@ +package com.alibaba.alicloud.oss.resource; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.common.utils.IOUtils; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.core.io.Resource; +import org.springframework.core.io.WritableResource; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.util.StreamUtils; + +import java.io.*; +import java.nio.charset.Charset; +import java.util.Random; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; + +/** + * @author lich + * @date 2019/8/29 + */ + +@SpringBootTest +@RunWith(SpringRunner.class) +public class OssStorageResourceTest { + + /** + * Used to test exception messages and types. + */ + @Rule + public ExpectedException expectedEx = ExpectedException.none(); + + @Autowired + private OSS oss; + + @Value("oss://aliyun-test-bucket/") + private Resource bucketResource; + + @Value("oss://aliyun-test-bucket/myfilekey") + private Resource remoteResource; + + public static byte[] generateRandomBytes(int blen) { + byte[] array = new byte[blen]; + new Random().nextBytes(array); + return array; + } + + @Test + public void testResourceType() { + assertEquals(OssStorageResource.class, remoteResource.getClass()); + OssStorageResource ossStorageResource = (OssStorageResource)remoteResource; + assertEquals("myfilekey", ossStorageResource.getFilename()); + assertFalse(ossStorageResource.isBucket()); + } + + @Test + public void testValidObject() throws Exception { + assertTrue(remoteResource.exists()); + OssStorageResource ossStorageResource = (OssStorageResource)remoteResource; + assertTrue(ossStorageResource.bucketExists()); + assertEquals(4096L, remoteResource.contentLength()); + assertEquals("oss://aliyun-test-bucket/myfilekey", remoteResource.getURI().toString()); + assertEquals("myfilekey", remoteResource.getFilename()); + } + + @Test + public void testBucketResource() throws Exception { + assertTrue(bucketResource.exists()); + assertTrue(((OssStorageResource)this.bucketResource).isBucket()); + assertTrue(((OssStorageResource)this.bucketResource).bucketExists()); + assertEquals("oss://aliyun-test-bucket/", bucketResource.getURI().toString()); + assertEquals("aliyun-test-bucket", this.bucketResource.getFilename()); + } + + @Test + public void testBucketNotEndingInSlash() { + assertTrue(new OssStorageResource(this.oss, "oss://aliyun-test-bucket").isBucket()); + } + + @Test + public void testSpecifyPathCorrect() { + OssStorageResource ossStorageResource = new OssStorageResource ( + this.oss, "oss://aliyun-test-bucket/myfilekey", false); + + assertTrue(ossStorageResource.exists()); + } + + @Test + public void testSpecifyBucketCorrect() { + OssStorageResource ossStorageResource = new OssStorageResource( + this.oss, "oss://aliyun-test-bucket", false); + + assertTrue(ossStorageResource.isBucket()); + assertEquals("aliyun-test-bucket", ossStorageResource.getBucket().getName()); + assertTrue(ossStorageResource.exists()); + } + + @Test + public void testBucketOutputStream() throws IOException { + this.expectedEx.expect(IllegalStateException.class); + this.expectedEx.expectMessage("Cannot open an output stream to a bucket: 'oss://aliyun-test-bucket/'"); + ((WritableResource) this.bucketResource).getOutputStream(); + } + + @Test + public void testBucketInputStream() throws IOException { + this.expectedEx.expect(IllegalStateException.class); + this.expectedEx.expectMessage("Cannot open an input stream to a bucket: 'oss://aliyun-test-bucket/'"); + this.bucketResource.getInputStream(); + } + + @Test + public void testBucketContentLength() throws IOException { + this.expectedEx.expect(FileNotFoundException.class); + this.expectedEx.expectMessage("OSSObject not existed."); + this.bucketResource.contentLength(); + } + + @Test + public void testBucketFile() throws IOException { + this.expectedEx.expect(UnsupportedOperationException.class); + this.expectedEx.expectMessage("oss://aliyun-test-bucket/ cannot be resolved to absolute file path"); + this.bucketResource.getFile(); + } + + @Test + public void testBucketLastModified() throws IOException { + this.expectedEx.expect(FileNotFoundException.class); + this.expectedEx.expectMessage("OSSObject not existed."); + this.bucketResource.lastModified(); + } + + @Test + public void testBucketResourceStatuses() { + assertFalse(this.bucketResource.isOpen()); + assertFalse(((WritableResource) this.bucketResource).isWritable()); + assertTrue(this.bucketResource.exists()); + } + + @Test + public void testWritable() throws Exception { + assertTrue(this.remoteResource instanceof WritableResource); + WritableResource writableResource = (WritableResource) this.remoteResource; + assertTrue(writableResource.isWritable()); + writableResource.getOutputStream(); + } + + @Test + public void testWritableOutputStream() throws Exception { + String location = "oss://aliyun-test-bucket/test"; + OssStorageResource resource = new OssStorageResource(this.oss, location, true); + OutputStream os = resource.getOutputStream(); + assertNotNull(os); + + byte[] randomBytes = generateRandomBytes(1203); + String expectedString = new String(randomBytes); + + os.write(randomBytes); + os.close(); + + InputStream in = resource.getInputStream(); + + byte[] result = StreamUtils.copyToByteArray(in); + String actualString = new String(result); + + assertEquals(expectedString, actualString); + } + + + + /** + * Configuration for the tests. + */ + @Configuration + @Import(OssStorageProtocolResolver.class) + static class TestConfiguration { + + @Bean + public static OSS mockOSS() { + DummyOssClient dummyOssStub = new DummyOssClient(); + OSS oss = mock(OSS.class); + + doAnswer(invocation -> + dummyOssStub.putObject( + invocation.getArgument(0), + invocation.getArgument(1), + invocation.getArgument(2) + )) + .when(oss).putObject(Mockito.anyString(), Mockito.anyString(), Mockito.any(InputStream.class)); + + doAnswer(invocation -> + dummyOssStub.getOSSObject( + invocation.getArgument(0), + invocation.getArgument(1) + )) + .when(oss).getObject(Mockito.anyString(), Mockito.anyString()); + + doAnswer(invocation -> dummyOssStub.bucketList()) + .when(oss).listBuckets(); + + doAnswer(invocation -> dummyOssStub.createBucket(invocation.getArgument(0))) + .when(oss).createBucket(Mockito.anyString()); + + // prepare object + dummyOssStub.createBucket("aliyun-test-bucket"); + + byte[] content = generateRandomBytes(4096); + ByteArrayInputStream inputStream = new ByteArrayInputStream(content); + dummyOssStub.putObject("aliyun-test-bucket", "myfilekey", inputStream); + + return oss; + } + + } + +} From 0de7efabd42e5f9bfd638b94d351a51cbcba6010 Mon Sep 17 00:00:00 2001 From: lichen782 Date: Tue, 10 Sep 2019 14:24:00 +0800 Subject: [PATCH 2/6] Add missing Apache 2 License --- .../oss/resource/OssStorageResource.java | 35 +- .../alicloud/oss/resource/DummyOssClient.java | 118 ++++--- .../oss/resource/OssStorageResourceTest.java | 329 +++++++++--------- 3 files changed, 264 insertions(+), 218 deletions(-) diff --git a/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageResource.java b/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageResource.java index 957411c8..2aeab0e0 100644 --- a/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageResource.java +++ b/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageResource.java @@ -216,6 +216,10 @@ public class OssStorageResource implements WritableResource { } } + /** + * create a bucket. + * @return + */ public Bucket createBucket() { return this.oss.createBucket(this.bucketName); } @@ -226,33 +230,37 @@ public class OssStorageResource implements WritableResource { } /** - * 获取一个OutputStream用于写操作。 - * 注意：写完成后必须关闭该流 + * acquire an OutputStream for write. + * Note: please close the stream after writing is done * @return - * @throws IOException + * @throws IOException */ @Override public OutputStream getOutputStream() throws IOException { if (isBucket()) { throw new IllegalStateException( - "Cannot open an output stream to a bucket: '" + getURI() + "'"); + "Cannot open an output stream to a bucket: '" + getURI() + "'"); } else { OSSObject ossObject; - try { + try { ossObject = this.getOSSObject(); - } catch (OSSException ex) { - if (ex.getMessage() != null && ex.getMessage().startsWith(MESSAGE_KEY_NOT_EXIST)) { - ossObject = null; - } else { - throw ex; + } + catch (OSSException ex) { + if (ex.getMessage() != null + && ex.getMessage().startsWith(MESSAGE_KEY_NOT_EXIST)) { + ossObject = null; + } + else { + throw ex; } } - if (ossObject == null ) { + if (ossObject == null) { if (!this.autoCreateFiles) { - throw new FileNotFoundException("The object was not found: " + getURI()); + throw new FileNotFoundException( + "The object was not found: " + getURI()); } } @@ -263,7 +271,8 @@ public class OssStorageResource implements WritableResource { executorService.submit(() -> { try { OssStorageResource.this.oss.putObject(bucketName, objectKey, in); - } catch (Exception ex) { + } + catch (Exception ex) { logger.error("Failed to put object", ex); } }); diff --git a/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/DummyOssClient.java b/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/DummyOssClient.java index 54b52278..d04a574a 100644 --- a/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/DummyOssClient.java +++ b/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/DummyOssClient.java @@ -1,3 +1,19 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.alibaba.alicloud.oss.resource; import com.aliyun.oss.model.Bucket; @@ -14,67 +30,69 @@ import java.util.concurrent.ConcurrentHashMap; /** * @author lich - * @date 2019/8/30 */ public class DummyOssClient { - private Map storeMap = new ConcurrentHashMap<>(); + private Map storeMap = new ConcurrentHashMap<>(); - private Map bucketSet = new HashMap<>(); + private Map bucketSet = new HashMap<>(); - public String getStoreKey(String bucketName, String objectKey) { - return String.join(".", bucketName, objectKey); - } + public String getStoreKey(String bucketName, String objectKey) { + return String.join(".", bucketName, objectKey); + } - public PutObjectResult putObject(String bucketName, String objectKey, InputStream inputStream) { + public PutObjectResult putObject(String bucketName, String objectKey, + InputStream inputStream) { - try { - byte[] result = StreamUtils.copyToByteArray(inputStream); - storeMap.put(getStoreKey(bucketName, objectKey), result); - } catch (IOException ex) { - throw new RuntimeException(ex); - } finally { - try { - inputStream.close(); - } catch (IOException ex) { - throw new RuntimeException(ex); - } - } + try { + byte[] result = StreamUtils.copyToByteArray(inputStream); + storeMap.put(getStoreKey(bucketName, objectKey), result); + } + catch (IOException ex) { + throw new RuntimeException(ex); + } + finally { + try { + inputStream.close(); + } + catch (IOException ex) { + throw new RuntimeException(ex); + } + } + return new PutObjectResult(); + } - return new PutObjectResult(); - } + public OSSObject getOSSObject(String bucketName, String objectKey) { + byte[] value = storeMap.get(this.getStoreKey(bucketName, objectKey)); + if (value == null) { + return null; + } + OSSObject ossObject = new OSSObject(); + ossObject.setBucketName(bucketName); + ossObject.setKey(objectKey); + InputStream inputStream = new ByteArrayInputStream(value); + ossObject.setObjectContent(inputStream); - public OSSObject getOSSObject(String bucketName, String objectKey) { - byte[] value = storeMap.get(this.getStoreKey(bucketName, objectKey)); - if (value == null) { - return null; - } - OSSObject ossObject = new OSSObject(); - ossObject.setBucketName(bucketName); - ossObject.setKey(objectKey); - InputStream inputStream = new ByteArrayInputStream(value); - ossObject.setObjectContent(inputStream); + ObjectMetadata objectMetadata = new ObjectMetadata(); + objectMetadata.setContentLength(value.length); + ossObject.setObjectMetadata(objectMetadata); - ObjectMetadata objectMetadata = new ObjectMetadata(); - objectMetadata.setContentLength(value.length); - ossObject.setObjectMetadata(objectMetadata); + return ossObject; + } - return ossObject; - } + public Bucket createBucket(String bucketName) { + if (bucketSet.containsKey(bucketName)) { + return bucketSet.get(bucketName); + } + Bucket bucket = new Bucket(); + bucket.setCreationDate(new Date()); + bucket.setName(bucketName); + bucketSet.put(bucketName, bucket); + return bucket; + } - public Bucket createBucket(String bucketName) { - if (bucketSet.containsKey(bucketName)) { - return bucketSet.get(bucketName); - } - Bucket bucket = new Bucket(); - bucket.setCreationDate(new Date()); - bucket.setName(bucketName); - bucketSet.put(bucketName, bucket); - return bucket; - } - - public List bucketList() { - return new ArrayList<>(bucketSet.values()); - } + public List bucketList() { + return new ArrayList<>(bucketSet.values()); + } } diff --git a/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/OssStorageResourceTest.java b/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/OssStorageResourceTest.java index 9949636c..46496a66 100644 --- a/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/OssStorageResourceTest.java +++ b/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/OssStorageResourceTest.java @@ -1,7 +1,22 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.alibaba.alicloud.oss.resource; import com.aliyun.oss.OSS; -import com.aliyun.oss.common.utils.IOUtils; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -19,7 +34,6 @@ import org.springframework.test.context.junit4.SpringRunner; import org.springframework.util.StreamUtils; import java.io.*; -import java.nio.charset.Charset; import java.util.Random; import static org.junit.Assert.*; @@ -28,200 +42,205 @@ import static org.mockito.Mockito.mock; /** * @author lich - * @date 2019/8/29 */ - @SpringBootTest @RunWith(SpringRunner.class) public class OssStorageResourceTest { - /** - * Used to test exception messages and types. - */ - @Rule - public ExpectedException expectedEx = ExpectedException.none(); + /** + * Used to test exception messages and types. + */ + @Rule + public ExpectedException expectedEx = ExpectedException.none(); - @Autowired - private OSS oss; + @Autowired + private OSS oss; - @Value("oss://aliyun-test-bucket/") - private Resource bucketResource; + @Value("oss://aliyun-test-bucket/") + private Resource bucketResource; - @Value("oss://aliyun-test-bucket/myfilekey") - private Resource remoteResource; + @Value("oss://aliyun-test-bucket/myfilekey") + private Resource remoteResource; - public static byte[] generateRandomBytes(int blen) { - byte[] array = new byte[blen]; - new Random().nextBytes(array); - return array; - } + public static byte[] generateRandomBytes(int blen) { + byte[] array = new byte[blen]; + new Random().nextBytes(array); + return array; + } - @Test - public void testResourceType() { - assertEquals(OssStorageResource.class, remoteResource.getClass()); - OssStorageResource ossStorageResource = (OssStorageResource)remoteResource; - assertEquals("myfilekey", ossStorageResource.getFilename()); - assertFalse(ossStorageResource.isBucket()); - } + @Test + public void testResourceType() { + assertEquals(OssStorageResource.class, remoteResource.getClass()); + OssStorageResource ossStorageResource = (OssStorageResource) remoteResource; + assertEquals("myfilekey", ossStorageResource.getFilename()); + assertFalse(ossStorageResource.isBucket()); + } - @Test - public void testValidObject() throws Exception { - assertTrue(remoteResource.exists()); - OssStorageResource ossStorageResource = (OssStorageResource)remoteResource; - assertTrue(ossStorageResource.bucketExists()); - assertEquals(4096L, remoteResource.contentLength()); - assertEquals("oss://aliyun-test-bucket/myfilekey", remoteResource.getURI().toString()); - assertEquals("myfilekey", remoteResource.getFilename()); - } + @Test + public void testValidObject() throws Exception { + assertTrue(remoteResource.exists()); + OssStorageResource ossStorageResource = (OssStorageResource) remoteResource; + assertTrue(ossStorageResource.bucketExists()); + assertEquals(4096L, remoteResource.contentLength()); + assertEquals("oss://aliyun-test-bucket/myfilekey", + remoteResource.getURI().toString()); + assertEquals("myfilekey", remoteResource.getFilename()); + } - @Test - public void testBucketResource() throws Exception { - assertTrue(bucketResource.exists()); - assertTrue(((OssStorageResource)this.bucketResource).isBucket()); - assertTrue(((OssStorageResource)this.bucketResource).bucketExists()); - assertEquals("oss://aliyun-test-bucket/", bucketResource.getURI().toString()); - assertEquals("aliyun-test-bucket", this.bucketResource.getFilename()); - } + @Test + public void testBucketResource() throws Exception { + assertTrue(bucketResource.exists()); + assertTrue(((OssStorageResource) this.bucketResource).isBucket()); + assertTrue(((OssStorageResource) this.bucketResource).bucketExists()); + assertEquals("oss://aliyun-test-bucket/", bucketResource.getURI().toString()); + assertEquals("aliyun-test-bucket", this.bucketResource.getFilename()); + } - @Test - public void testBucketNotEndingInSlash() { - assertTrue(new OssStorageResource(this.oss, "oss://aliyun-test-bucket").isBucket()); - } + @Test + public void testBucketNotEndingInSlash() { + assertTrue( + new OssStorageResource(this.oss, "oss://aliyun-test-bucket").isBucket()); + } - @Test - public void testSpecifyPathCorrect() { - OssStorageResource ossStorageResource = new OssStorageResource ( - this.oss, "oss://aliyun-test-bucket/myfilekey", false); + @Test + public void testSpecifyPathCorrect() { + OssStorageResource ossStorageResource = new OssStorageResource(this.oss, + "oss://aliyun-test-bucket/myfilekey", false); - assertTrue(ossStorageResource.exists()); - } + assertTrue(ossStorageResource.exists()); + } - @Test - public void testSpecifyBucketCorrect() { - OssStorageResource ossStorageResource = new OssStorageResource( - this.oss, "oss://aliyun-test-bucket", false); + @Test + public void testSpecifyBucketCorrect() { + OssStorageResource ossStorageResource = new OssStorageResource(this.oss, + "oss://aliyun-test-bucket", false); - assertTrue(ossStorageResource.isBucket()); - assertEquals("aliyun-test-bucket", ossStorageResource.getBucket().getName()); - assertTrue(ossStorageResource.exists()); - } + assertTrue(ossStorageResource.isBucket()); + assertEquals("aliyun-test-bucket", ossStorageResource.getBucket().getName()); + assertTrue(ossStorageResource.exists()); + } - @Test - public void testBucketOutputStream() throws IOException { - this.expectedEx.expect(IllegalStateException.class); - this.expectedEx.expectMessage("Cannot open an output stream to a bucket: 'oss://aliyun-test-bucket/'"); - ((WritableResource) this.bucketResource).getOutputStream(); - } + @Test + public void testBucketOutputStream() throws IOException { + this.expectedEx.expect(IllegalStateException.class); + this.expectedEx.expectMessage( + "Cannot open an output stream to a bucket: 'oss://aliyun-test-bucket/'"); + ((WritableResource) this.bucketResource).getOutputStream(); + } - @Test - public void testBucketInputStream() throws IOException { - this.expectedEx.expect(IllegalStateException.class); - this.expectedEx.expectMessage("Cannot open an input stream to a bucket: 'oss://aliyun-test-bucket/'"); - this.bucketResource.getInputStream(); - } + @Test + public void testBucketInputStream() throws IOException { + this.expectedEx.expect(IllegalStateException.class); + this.expectedEx.expectMessage( + "Cannot open an input stream to a bucket: 'oss://aliyun-test-bucket/'"); + this.bucketResource.getInputStream(); + } - @Test - public void testBucketContentLength() throws IOException { - this.expectedEx.expect(FileNotFoundException.class); - this.expectedEx.expectMessage("OSSObject not existed."); - this.bucketResource.contentLength(); - } + @Test + public void testBucketContentLength() throws IOException { + this.expectedEx.expect(FileNotFoundException.class); + this.expectedEx.expectMessage("OSSObject not existed."); + this.bucketResource.contentLength(); + } - @Test - public void testBucketFile() throws IOException { - this.expectedEx.expect(UnsupportedOperationException.class); - this.expectedEx.expectMessage("oss://aliyun-test-bucket/ cannot be resolved to absolute file path"); - this.bucketResource.getFile(); - } + @Test + public void testBucketFile() throws IOException { + this.expectedEx.expect(UnsupportedOperationException.class); + this.expectedEx.expectMessage( + "oss://aliyun-test-bucket/ cannot be resolved to absolute file path"); + this.bucketResource.getFile(); + } - @Test - public void testBucketLastModified() throws IOException { - this.expectedEx.expect(FileNotFoundException.class); - this.expectedEx.expectMessage("OSSObject not existed."); - this.bucketResource.lastModified(); - } + @Test + public void testBucketLastModified() throws IOException { + this.expectedEx.expect(FileNotFoundException.class); + this.expectedEx.expectMessage("OSSObject not existed."); + this.bucketResource.lastModified(); + } - @Test - public void testBucketResourceStatuses() { - assertFalse(this.bucketResource.isOpen()); - assertFalse(((WritableResource) this.bucketResource).isWritable()); - assertTrue(this.bucketResource.exists()); - } + @Test + public void testBucketResourceStatuses() { + assertFalse(this.bucketResource.isOpen()); + assertFalse(((WritableResource) this.bucketResource).isWritable()); + assertTrue(this.bucketResource.exists()); + } - @Test - public void testWritable() throws Exception { - assertTrue(this.remoteResource instanceof WritableResource); - WritableResource writableResource = (WritableResource) this.remoteResource; - assertTrue(writableResource.isWritable()); - writableResource.getOutputStream(); - } + @Test + public void testWritable() throws Exception { + assertTrue(this.remoteResource instanceof WritableResource); + WritableResource writableResource = (WritableResource) this.remoteResource; + assertTrue(writableResource.isWritable()); + writableResource.getOutputStream(); + } - @Test - public void testWritableOutputStream() throws Exception { - String location = "oss://aliyun-test-bucket/test"; - OssStorageResource resource = new OssStorageResource(this.oss, location, true); - OutputStream os = resource.getOutputStream(); - assertNotNull(os); + @Test + public void testWritableOutputStream() throws Exception { + String location = "oss://aliyun-test-bucket/test"; + OssStorageResource resource = new OssStorageResource(this.oss, location, true); + OutputStream os = resource.getOutputStream(); + assertNotNull(os); - byte[] randomBytes = generateRandomBytes(1203); - String expectedString = new String(randomBytes); + byte[] randomBytes = generateRandomBytes(1203); + String expectedString = new String(randomBytes); - os.write(randomBytes); - os.close(); + os.write(randomBytes); + os.close(); - InputStream in = resource.getInputStream(); + InputStream in = resource.getInputStream(); - byte[] result = StreamUtils.copyToByteArray(in); - String actualString = new String(result); + byte[] result = StreamUtils.copyToByteArray(in); + String actualString = new String(result); - assertEquals(expectedString, actualString); - } + assertEquals(expectedString, actualString); + } + @Test + public void testCreateBucket() { + String location = "oss://my-new-test-bucket/"; + OssStorageResource resource = new OssStorageResource(this.oss, location, true); + resource.createBucket(); - /** - * Configuration for the tests. - */ - @Configuration - @Import(OssStorageProtocolResolver.class) - static class TestConfiguration { + assertTrue(resource.bucketExists()); - @Bean - public static OSS mockOSS() { - DummyOssClient dummyOssStub = new DummyOssClient(); - OSS oss = mock(OSS.class); + } - doAnswer(invocation -> - dummyOssStub.putObject( - invocation.getArgument(0), - invocation.getArgument(1), - invocation.getArgument(2) - )) - .when(oss).putObject(Mockito.anyString(), Mockito.anyString(), Mockito.any(InputStream.class)); + /** + * Configuration for the tests. + */ + @Configuration + @Import(OssStorageProtocolResolver.class) + static class TestConfiguration { - doAnswer(invocation -> - dummyOssStub.getOSSObject( - invocation.getArgument(0), - invocation.getArgument(1) - )) - .when(oss).getObject(Mockito.anyString(), Mockito.anyString()); + @Bean + public static OSS mockOSS() { + DummyOssClient dummyOssStub = new DummyOssClient(); + OSS oss = mock(OSS.class); - doAnswer(invocation -> dummyOssStub.bucketList()) - .when(oss).listBuckets(); + doAnswer(invocation -> dummyOssStub.putObject(invocation.getArgument(0), + invocation.getArgument(1), invocation.getArgument(2))).when(oss) + .putObject(Mockito.anyString(), Mockito.anyString(), + Mockito.any(InputStream.class)); - doAnswer(invocation -> dummyOssStub.createBucket(invocation.getArgument(0))) - .when(oss).createBucket(Mockito.anyString()); + doAnswer(invocation -> dummyOssStub.getOSSObject(invocation.getArgument(0), + invocation.getArgument(1))).when(oss).getObject(Mockito.anyString(), + Mockito.anyString()); - // prepare object - dummyOssStub.createBucket("aliyun-test-bucket"); + doAnswer(invocation -> dummyOssStub.bucketList()).when(oss).listBuckets(); - byte[] content = generateRandomBytes(4096); - ByteArrayInputStream inputStream = new ByteArrayInputStream(content); - dummyOssStub.putObject("aliyun-test-bucket", "myfilekey", inputStream); + doAnswer(invocation -> dummyOssStub.createBucket(invocation.getArgument(0))) + .when(oss).createBucket(Mockito.anyString()); - return oss; - } + // prepare object + dummyOssStub.createBucket("aliyun-test-bucket"); - } + byte[] content = generateRandomBytes(4096); + ByteArrayInputStream inputStream = new ByteArrayInputStream(content); + dummyOssStub.putObject("aliyun-test-bucket", "myfilekey", inputStream); + + return oss; + } + + } } From 572b13a0b3cbe1c20856c43bf1f2c784cea93a91 Mon Sep 17 00:00:00 2001 From: lichen782 Date: Tue, 10 Sep 2019 14:55:51 +0800 Subject: [PATCH 3/6] Refactor the OssController upload2 to use try with clause --- .../alibaba/cloud/examples/OssController.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/spring-cloud-alibaba-examples/oss-example/src/main/java/com/alibaba/cloud/examples/OssController.java b/spring-cloud-alibaba-examples/oss-example/src/main/java/com/alibaba/cloud/examples/OssController.java index 30d51c72..7dc364cb 100644 --- a/spring-cloud-alibaba-examples/oss-example/src/main/java/com/alibaba/cloud/examples/OssController.java +++ b/spring-cloud-alibaba-examples/oss-example/src/main/java/com/alibaba/cloud/examples/OssController.java @@ -1,6 +1,5 @@ package com.alibaba.cloud.examples; -import com.alibaba.alicloud.oss.resource.OssStorageResource; import com.aliyun.oss.OSS; import com.aliyun.oss.common.utils.IOUtils; import com.aliyun.oss.model.OSSObject; @@ -8,6 +7,7 @@ import org.apache.commons.codec.CharEncoding; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.Resource; +import org.springframework.core.io.WritableResource; import org.springframework.util.StreamUtils; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RestController; @@ -74,15 +74,14 @@ public class OssController { @GetMapping("/upload2") public String uploadWithOutputStream() { - OssStorageResource ossStorageResource = new OssStorageResource(this.ossClient, - "oss://" + OssApplication.BUCKET_NAME + "/oss-test.json", true); try { - InputStream inputStream = localFile.getInputStream(); - OutputStream outputStream = ossStorageResource.getOutputStream(); - StreamUtils.copy(inputStream, outputStream); - inputStream.close(); - outputStream.close(); - } catch (Exception ex) { + try (OutputStream outputStream = ((WritableResource) this.remoteFile) + .getOutputStream(); + InputStream inputStream = localFile.getInputStream()) { + StreamUtils.copy(inputStream, outputStream); + } + } + catch (Exception ex) { ex.printStackTrace(); return "upload with outputStream failed"; } From fdf8bda86bf13890d40df7c7b400d481c0b1305e Mon Sep 17 00:00:00 2001 From: lichen782 Date: Tue, 10 Sep 2019 15:51:22 +0800 Subject: [PATCH 4/6] Use common execute service for all oss resource --- .../alibaba/alicloud/oss/resource/OssStorageResource.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageResource.java b/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageResource.java index 2aeab0e0..215c6bab 100644 --- a/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageResource.java +++ b/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageResource.java @@ -58,7 +58,8 @@ public class OssStorageResource implements WritableResource { private final URI location; private final boolean autoCreateFiles; - private final ExecutorService executorService; + private static final ExecutorService executorService = new ThreadPoolExecutor(8, 128, + 60, TimeUnit.SECONDS, new SynchronousQueue<>()); public OssStorageResource(OSS oss, String location) { this(oss, location, false); @@ -86,9 +87,6 @@ public class OssStorageResource implements WritableResource { throw new IllegalArgumentException("Invalid location: " + location, e); } - this.executorService = new ThreadPoolExecutor( - 1, 1, 60, TimeUnit.SECONDS, - new SynchronousQueue<>()); } public boolean isAutoCreateFiles() { From b83b1b78dcc88e3967d51f871c83bcf2c1ee5999 Mon Sep 17 00:00:00 2001 From: lichen782 Date: Mon, 16 Sep 2019 16:09:27 +0800 Subject: [PATCH 5/6] Use bean factory to search executor service instead of hardcode it. --- .../alicloud/oss/OssAutoConfiguration.java | 12 ++++++++ .../alibaba/alicloud/oss/OssConstants.java | 2 ++ .../resource/OssStorageProtocolResolver.java | 2 +- .../oss/resource/OssStorageResource.java | 23 +++++++------- .../oss/resource/OssStorageResourceTest.java | 30 +++++++++++++++---- 5 files changed, 53 insertions(+), 16 deletions(-) diff --git a/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/OssAutoConfiguration.java b/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/OssAutoConfiguration.java index 0cbf049e..d5e8052b 100644 --- a/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/OssAutoConfiguration.java +++ b/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/OssAutoConfiguration.java @@ -25,6 +25,11 @@ import org.springframework.context.annotation.Configuration; import com.alibaba.alicloud.oss.resource.OssStorageProtocolResolver; import com.aliyun.oss.OSS; +import org.springframework.context.annotation.Lazy; + +import java.util.concurrent.*; + +import static com.alibaba.alicloud.oss.OssConstants.OSS_TASK_EXECUTOR_BEAN_NAME; /** * OSS Auto {@link Configuration} @@ -42,4 +47,11 @@ public class OssAutoConfiguration { return new OssStorageProtocolResolver(); } + @Bean(name = OSS_TASK_EXECUTOR_BEAN_NAME) + @ConditionalOnMissingBean + public ExecutorService ossTaskExecutor() { + return new ThreadPoolExecutor(8, 128, + 60, TimeUnit.SECONDS, new SynchronousQueue<>()); + } + } diff --git a/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/OssConstants.java b/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/OssConstants.java index d018b892..8384092a 100644 --- a/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/OssConstants.java +++ b/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/OssConstants.java @@ -26,4 +26,6 @@ public interface OssConstants { String PREFIX = "spring.cloud.alibaba.oss"; String ENABLED = PREFIX + ".enabled"; + String OSS_TASK_EXECUTOR_BEAN_NAME = "ossTaskExecutor"; + } diff --git a/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageProtocolResolver.java b/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageProtocolResolver.java index 2d1f2d50..eb611163 100644 --- a/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageProtocolResolver.java +++ b/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageProtocolResolver.java @@ -63,7 +63,7 @@ public class OssStorageProtocolResolver if (!location.startsWith(PROTOCOL)) { return null; } - return new OssStorageResource(getOSS(), location); + return new OssStorageResource(getOSS(), location, beanFactory); } @Override diff --git a/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageResource.java b/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageResource.java index 215c6bab..64e242f4 100644 --- a/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageResource.java +++ b/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageResource.java @@ -23,6 +23,7 @@ import com.aliyun.oss.model.Bucket; import com.aliyun.oss.model.OSSObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; import org.springframework.core.io.Resource; import org.springframework.core.io.WritableResource; import org.springframework.util.Assert; @@ -32,9 +33,8 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.concurrent.ExecutorService; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; + +import static com.alibaba.alicloud.oss.OssConstants.OSS_TASK_EXECUTOR_BEAN_NAME; /** * Implements {@link Resource} for reading and writing objects in Aliyun Object Storage @@ -58,19 +58,21 @@ public class OssStorageResource implements WritableResource { private final URI location; private final boolean autoCreateFiles; - private static final ExecutorService executorService = new ThreadPoolExecutor(8, 128, - 60, TimeUnit.SECONDS, new SynchronousQueue<>()); + private final ExecutorService ossTaskExecutor; - public OssStorageResource(OSS oss, String location) { - this(oss, location, false); + private final ConfigurableListableBeanFactory beanFactory; + + public OssStorageResource(OSS oss, String location, ConfigurableListableBeanFactory beanFactory) { + this(oss, location, beanFactory,false); } - public OssStorageResource(OSS oss, String location, boolean autoCreateFiles) { + public OssStorageResource(OSS oss, String location,ConfigurableListableBeanFactory beanFactory, boolean autoCreateFiles) { Assert.notNull(oss, "Object Storage Service can not be null"); Assert.isTrue(location.startsWith(OssStorageProtocolResolver.PROTOCOL), "Location must start with " + OssStorageProtocolResolver.PROTOCOL); this.oss = oss; this.autoCreateFiles = autoCreateFiles; + this.beanFactory = beanFactory; try { URI locationUri = new URI(location); this.bucketName = locationUri.getAuthority(); @@ -87,6 +89,7 @@ public class OssStorageResource implements WritableResource { throw new IllegalArgumentException("Invalid location: " + location, e); } + this.ossTaskExecutor = this.beanFactory.getBean(OSS_TASK_EXECUTOR_BEAN_NAME, ExecutorService.class); } public boolean isAutoCreateFiles() { @@ -146,7 +149,7 @@ public class OssStorageResource implements WritableResource { @Override public Resource createRelative(String relativePath) throws IOException { return new OssStorageResource(this.oss, - this.location.resolve(relativePath).toString()); + this.location.resolve(relativePath).toString(), this.beanFactory); } @Override @@ -266,7 +269,7 @@ public class OssStorageResource implements WritableResource { PipedInputStream in = new PipedInputStream(); final PipedOutputStream out = new PipedOutputStream(in); - executorService.submit(() -> { + ossTaskExecutor.submit(() -> { try { OssStorageResource.this.oss.putObject(bucketName, objectKey, in); } diff --git a/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/OssStorageResourceTest.java b/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/OssStorageResourceTest.java index 46496a66..b57eab48 100644 --- a/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/OssStorageResourceTest.java +++ b/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/OssStorageResourceTest.java @@ -22,8 +22,12 @@ import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; import org.mockito.Mockito; +import org.springframework.beans.BeansException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.config.BeanFactoryPostProcessor; +import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -35,7 +39,12 @@ import org.springframework.util.StreamUtils; import java.io.*; import java.util.Random; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import static com.alibaba.alicloud.oss.OssConstants.OSS_TASK_EXECUTOR_BEAN_NAME; import static org.junit.Assert.*; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -53,6 +62,9 @@ public class OssStorageResourceTest { @Rule public ExpectedException expectedEx = ExpectedException.none(); + @Autowired + private ConfigurableListableBeanFactory beanFactory; + @Autowired private OSS oss; @@ -99,13 +111,14 @@ public class OssStorageResourceTest { @Test public void testBucketNotEndingInSlash() { assertTrue( - new OssStorageResource(this.oss, "oss://aliyun-test-bucket").isBucket()); + new OssStorageResource(this.oss, "oss://aliyun-test-bucket", beanFactory) + .isBucket()); } @Test public void testSpecifyPathCorrect() { OssStorageResource ossStorageResource = new OssStorageResource(this.oss, - "oss://aliyun-test-bucket/myfilekey", false); + "oss://aliyun-test-bucket/myfilekey", beanFactory, false); assertTrue(ossStorageResource.exists()); } @@ -113,7 +126,7 @@ public class OssStorageResourceTest { @Test public void testSpecifyBucketCorrect() { OssStorageResource ossStorageResource = new OssStorageResource(this.oss, - "oss://aliyun-test-bucket", false); + "oss://aliyun-test-bucket", beanFactory, false); assertTrue(ossStorageResource.isBucket()); assertEquals("aliyun-test-bucket", ossStorageResource.getBucket().getName()); @@ -176,7 +189,7 @@ public class OssStorageResourceTest { @Test public void testWritableOutputStream() throws Exception { String location = "oss://aliyun-test-bucket/test"; - OssStorageResource resource = new OssStorageResource(this.oss, location, true); + OssStorageResource resource = new OssStorageResource(this.oss, location, beanFactory,true); OutputStream os = resource.getOutputStream(); assertNotNull(os); @@ -197,7 +210,7 @@ public class OssStorageResourceTest { @Test public void testCreateBucket() { String location = "oss://my-new-test-bucket/"; - OssStorageResource resource = new OssStorageResource(this.oss, location, true); + OssStorageResource resource = new OssStorageResource(this.oss, location, beanFactory, true); resource.createBucket(); @@ -212,6 +225,13 @@ public class OssStorageResourceTest { @Import(OssStorageProtocolResolver.class) static class TestConfiguration { + @Bean(name = OSS_TASK_EXECUTOR_BEAN_NAME) + @ConditionalOnMissingBean + public ExecutorService ossTaskExecutor() { + return new ThreadPoolExecutor(8, 128, + 60, TimeUnit.SECONDS, new SynchronousQueue<>()); + } + @Bean public static OSS mockOSS() { DummyOssClient dummyOssStub = new DummyOssClient(); From d195ebd299c0acab96a0c0c3eb18c3b748a29c92 Mon Sep 17 00:00:00 2001 From: lichen782 Date: Tue, 17 Sep 2019 09:26:08 +0800 Subject: [PATCH 6/6] Correct core size of the pool --- .../java/com/alibaba/alicloud/oss/OssAutoConfiguration.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/OssAutoConfiguration.java b/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/OssAutoConfiguration.java index d5e8052b..2392063b 100644 --- a/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/OssAutoConfiguration.java +++ b/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/OssAutoConfiguration.java @@ -50,8 +50,9 @@ public class OssAutoConfiguration { @Bean(name = OSS_TASK_EXECUTOR_BEAN_NAME) @ConditionalOnMissingBean public ExecutorService ossTaskExecutor() { - return new ThreadPoolExecutor(8, 128, - 60, TimeUnit.SECONDS, new SynchronousQueue<>()); + int coreSize = Runtime.getRuntime().availableProcessors(); + return new ThreadPoolExecutor(coreSize, 128, 60, TimeUnit.SECONDS, + new SynchronousQueue<>()); } }