diff --git a/spring-cloud-alibaba-examples/oss-example/src/main/java/com/alibaba/cloud/examples/OssController.java b/spring-cloud-alibaba-examples/oss-example/src/main/java/com/alibaba/cloud/examples/OssController.java index c1c9d261..7dc364cb 100644 --- a/spring-cloud-alibaba-examples/oss-example/src/main/java/com/alibaba/cloud/examples/OssController.java +++ b/spring-cloud-alibaba-examples/oss-example/src/main/java/com/alibaba/cloud/examples/OssController.java @@ -1,18 +1,20 @@ package com.alibaba.cloud.examples; -import java.nio.charset.Charset; - -import org.apache.commons.codec.CharEncoding; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.core.io.Resource; -import org.springframework.util.StreamUtils; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RestController; - import com.aliyun.oss.OSS; import com.aliyun.oss.common.utils.IOUtils; import com.aliyun.oss.model.OSSObject; +import org.apache.commons.codec.CharEncoding; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.io.Resource; +import org.springframework.core.io.WritableResource; +import org.springframework.util.StreamUtils; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.Charset; /** * OSS Controller @@ -25,8 +27,11 @@ public class OssController { @Autowired private OSS ossClient; + @Value("classpath:/oss-test.json") + private Resource localFile; + @Value("oss://" + OssApplication.BUCKET_NAME + "/oss-test.json") - private Resource file; + private Resource remoteFile; @GetMapping("/upload") public String upload() { @@ -45,7 +50,7 @@ public class OssController { public String fileResource() { try { return "get file resource success. content: " + StreamUtils.copyToString( - file.getInputStream(), Charset.forName(CharEncoding.UTF_8)); + remoteFile.getInputStream(), Charset.forName(CharEncoding.UTF_8)); } catch (Exception e) { e.printStackTrace(); @@ -67,4 +72,20 @@ public class OssController { } } + @GetMapping("/upload2") + public String uploadWithOutputStream() { + try { + try (OutputStream outputStream = ((WritableResource) this.remoteFile) + .getOutputStream(); + InputStream inputStream = localFile.getInputStream()) { + StreamUtils.copy(inputStream, outputStream); + } + } + catch (Exception ex) { + ex.printStackTrace(); + return "upload with outputStream failed"; + } + return "upload success"; + } + } diff --git a/spring-cloud-alicloud-oss/pom.xml b/spring-cloud-alicloud-oss/pom.xml index 12dfb67c..d69be7f6 100644 --- a/spring-cloud-alicloud-oss/pom.xml +++ b/spring-cloud-alicloud-oss/pom.xml @@ -58,6 +58,12 @@ test + + org.mockito + mockito-core + test + + diff --git a/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/OssAutoConfiguration.java b/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/OssAutoConfiguration.java index 0cbf049e..2392063b 100644 --- a/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/OssAutoConfiguration.java +++ b/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/OssAutoConfiguration.java @@ -25,6 +25,11 @@ import org.springframework.context.annotation.Configuration; import com.alibaba.alicloud.oss.resource.OssStorageProtocolResolver; import com.aliyun.oss.OSS; +import org.springframework.context.annotation.Lazy; + +import java.util.concurrent.*; + +import static com.alibaba.alicloud.oss.OssConstants.OSS_TASK_EXECUTOR_BEAN_NAME; /** * OSS Auto {@link Configuration} @@ -42,4 +47,12 @@ public class OssAutoConfiguration { return new OssStorageProtocolResolver(); } + @Bean(name = OSS_TASK_EXECUTOR_BEAN_NAME) + @ConditionalOnMissingBean + public ExecutorService ossTaskExecutor() { + int coreSize = Runtime.getRuntime().availableProcessors(); + return new ThreadPoolExecutor(coreSize, 128, 60, TimeUnit.SECONDS, + new SynchronousQueue<>()); + } + } diff --git a/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/OssConstants.java b/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/OssConstants.java index d018b892..8384092a 100644 --- a/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/OssConstants.java +++ b/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/OssConstants.java @@ -26,4 +26,6 @@ public interface OssConstants { String PREFIX = "spring.cloud.alibaba.oss"; String ENABLED = PREFIX + ".enabled"; + String OSS_TASK_EXECUTOR_BEAN_NAME = "ossTaskExecutor"; + } diff --git a/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageProtocolResolver.java b/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageProtocolResolver.java index 2d1f2d50..eb611163 100644 --- a/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageProtocolResolver.java +++ b/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageProtocolResolver.java @@ -63,7 +63,7 @@ public class OssStorageProtocolResolver if (!location.startsWith(PROTOCOL)) { return null; } - return new OssStorageResource(getOSS(), location); + return new OssStorageResource(getOSS(), location, beanFactory); } @Override diff --git a/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageResource.java b/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageResource.java index 70825bae..64e242f4 100644 --- a/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageResource.java +++ b/spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageResource.java @@ -16,22 +16,25 @@ package com.alibaba.alicloud.oss.resource; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; - -import org.springframework.core.io.Resource; -import org.springframework.util.Assert; - import com.aliyun.oss.ClientException; import com.aliyun.oss.OSS; import com.aliyun.oss.OSSException; import com.aliyun.oss.model.Bucket; import com.aliyun.oss.model.OSSObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; +import org.springframework.core.io.Resource; +import org.springframework.core.io.WritableResource; +import org.springframework.util.Assert; + +import java.io.*; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.concurrent.ExecutorService; + +import static com.alibaba.alicloud.oss.OssConstants.OSS_TASK_EXECUTOR_BEAN_NAME; /** * Implements {@link Resource} for reading and writing objects in Aliyun Object Storage @@ -43,18 +46,33 @@ import com.aliyun.oss.model.OSSObject; * @see Bucket * @see OSSObject */ -public class OssStorageResource implements Resource { +public class OssStorageResource implements WritableResource { + + private static final Logger logger = LoggerFactory.getLogger(OssStorageResource.class); + + private static final String MESSAGE_KEY_NOT_EXIST = "The specified key does not exist."; private final OSS oss; private final String bucketName; private final String objectKey; private final URI location; + private final boolean autoCreateFiles; - public OssStorageResource(OSS oss, String location) { + private final ExecutorService ossTaskExecutor; + + private final ConfigurableListableBeanFactory beanFactory; + + public OssStorageResource(OSS oss, String location, ConfigurableListableBeanFactory beanFactory) { + this(oss, location, beanFactory,false); + } + + public OssStorageResource(OSS oss, String location,ConfigurableListableBeanFactory beanFactory, boolean autoCreateFiles) { Assert.notNull(oss, "Object Storage Service can not be null"); Assert.isTrue(location.startsWith(OssStorageProtocolResolver.PROTOCOL), "Location must start with " + OssStorageProtocolResolver.PROTOCOL); this.oss = oss; + this.autoCreateFiles = autoCreateFiles; + this.beanFactory = beanFactory; try { URI locationUri = new URI(location); this.bucketName = locationUri.getAuthority(); @@ -70,6 +88,12 @@ public class OssStorageResource implements Resource { catch (URISyntaxException e) { throw new IllegalArgumentException("Invalid location: " + location, e); } + + this.ossTaskExecutor = this.beanFactory.getBean(OSS_TASK_EXECUTOR_BEAN_NAME, ExecutorService.class); + } + + public boolean isAutoCreateFiles() { + return this.autoCreateFiles; } @Override @@ -125,7 +149,7 @@ public class OssStorageResource implements Resource { @Override public Resource createRelative(String relativePath) throws IOException { return new OssStorageResource(this.oss, - this.location.resolve(relativePath).toString()); + this.location.resolve(relativePath).toString(), this.beanFactory); } @Override @@ -193,4 +217,70 @@ public class OssStorageResource implements Resource { } } + /** + * create a bucket. + * @return + */ + public Bucket createBucket() { + return this.oss.createBucket(this.bucketName); + } + + @Override + public boolean isWritable() { + return !isBucket() && (this.autoCreateFiles || exists()); + } + + /** + * acquire an OutputStream for write. + * Note: please close the stream after writing is done + * @return + * @throws IOException + */ + @Override + public OutputStream getOutputStream() throws IOException { + if (isBucket()) { + throw new IllegalStateException( + "Cannot open an output stream to a bucket: '" + getURI() + "'"); + } + else { + OSSObject ossObject; + + try { + ossObject = this.getOSSObject(); + } + catch (OSSException ex) { + if (ex.getMessage() != null + && ex.getMessage().startsWith(MESSAGE_KEY_NOT_EXIST)) { + ossObject = null; + } + else { + throw ex; + } + } + + if (ossObject == null) { + if (!this.autoCreateFiles) { + throw new FileNotFoundException( + "The object was not found: " + getURI()); + } + + } + + PipedInputStream in = new PipedInputStream(); + final PipedOutputStream out = new PipedOutputStream(in); + + ossTaskExecutor.submit(() -> { + try { + OssStorageResource.this.oss.putObject(bucketName, objectKey, in); + } + catch (Exception ex) { + logger.error("Failed to put object", ex); + } + }); + + return out; + } + + } + } diff --git a/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/DummyOssClient.java b/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/DummyOssClient.java new file mode 100644 index 00000000..d04a574a --- /dev/null +++ b/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/DummyOssClient.java @@ -0,0 +1,98 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.alicloud.oss.resource; + +import com.aliyun.oss.model.Bucket; +import com.aliyun.oss.model.OSSObject; +import com.aliyun.oss.model.ObjectMetadata; +import com.aliyun.oss.model.PutObjectResult; +import org.springframework.util.StreamUtils; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +/** + * @author lich + */ +public class DummyOssClient { + + private Map storeMap = new ConcurrentHashMap<>(); + + private Map bucketSet = new HashMap<>(); + + public String getStoreKey(String bucketName, String objectKey) { + return String.join(".", bucketName, objectKey); + } + + public PutObjectResult putObject(String bucketName, String objectKey, + InputStream inputStream) { + + try { + byte[] result = StreamUtils.copyToByteArray(inputStream); + storeMap.put(getStoreKey(bucketName, objectKey), result); + } + catch (IOException ex) { + throw new RuntimeException(ex); + } + finally { + try { + inputStream.close(); + } + catch (IOException ex) { + throw new RuntimeException(ex); + } + } + + return new PutObjectResult(); + } + + public OSSObject getOSSObject(String bucketName, String objectKey) { + byte[] value = storeMap.get(this.getStoreKey(bucketName, objectKey)); + if (value == null) { + return null; + } + OSSObject ossObject = new OSSObject(); + ossObject.setBucketName(bucketName); + ossObject.setKey(objectKey); + InputStream inputStream = new ByteArrayInputStream(value); + ossObject.setObjectContent(inputStream); + + ObjectMetadata objectMetadata = new ObjectMetadata(); + objectMetadata.setContentLength(value.length); + ossObject.setObjectMetadata(objectMetadata); + + return ossObject; + } + + public Bucket createBucket(String bucketName) { + if (bucketSet.containsKey(bucketName)) { + return bucketSet.get(bucketName); + } + Bucket bucket = new Bucket(); + bucket.setCreationDate(new Date()); + bucket.setName(bucketName); + bucketSet.put(bucketName, bucket); + return bucket; + } + + public List bucketList() { + return new ArrayList<>(bucketSet.values()); + } +} diff --git a/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/OssStorageResourceTest.java b/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/OssStorageResourceTest.java new file mode 100644 index 00000000..b57eab48 --- /dev/null +++ b/spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/OssStorageResourceTest.java @@ -0,0 +1,266 @@ +/* + * Copyright (C) 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.alicloud.oss.resource; + +import com.aliyun.oss.OSS; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.config.BeanFactoryPostProcessor; +import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.core.io.Resource; +import org.springframework.core.io.WritableResource; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.util.StreamUtils; + +import java.io.*; +import java.util.Random; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import static com.alibaba.alicloud.oss.OssConstants.OSS_TASK_EXECUTOR_BEAN_NAME; +import static org.junit.Assert.*; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; + +/** + * @author lich + */ +@SpringBootTest +@RunWith(SpringRunner.class) +public class OssStorageResourceTest { + + /** + * Used to test exception messages and types. + */ + @Rule + public ExpectedException expectedEx = ExpectedException.none(); + + @Autowired + private ConfigurableListableBeanFactory beanFactory; + + @Autowired + private OSS oss; + + @Value("oss://aliyun-test-bucket/") + private Resource bucketResource; + + @Value("oss://aliyun-test-bucket/myfilekey") + private Resource remoteResource; + + public static byte[] generateRandomBytes(int blen) { + byte[] array = new byte[blen]; + new Random().nextBytes(array); + return array; + } + + @Test + public void testResourceType() { + assertEquals(OssStorageResource.class, remoteResource.getClass()); + OssStorageResource ossStorageResource = (OssStorageResource) remoteResource; + assertEquals("myfilekey", ossStorageResource.getFilename()); + assertFalse(ossStorageResource.isBucket()); + } + + @Test + public void testValidObject() throws Exception { + assertTrue(remoteResource.exists()); + OssStorageResource ossStorageResource = (OssStorageResource) remoteResource; + assertTrue(ossStorageResource.bucketExists()); + assertEquals(4096L, remoteResource.contentLength()); + assertEquals("oss://aliyun-test-bucket/myfilekey", + remoteResource.getURI().toString()); + assertEquals("myfilekey", remoteResource.getFilename()); + } + + @Test + public void testBucketResource() throws Exception { + assertTrue(bucketResource.exists()); + assertTrue(((OssStorageResource) this.bucketResource).isBucket()); + assertTrue(((OssStorageResource) this.bucketResource).bucketExists()); + assertEquals("oss://aliyun-test-bucket/", bucketResource.getURI().toString()); + assertEquals("aliyun-test-bucket", this.bucketResource.getFilename()); + } + + @Test + public void testBucketNotEndingInSlash() { + assertTrue( + new OssStorageResource(this.oss, "oss://aliyun-test-bucket", beanFactory) + .isBucket()); + } + + @Test + public void testSpecifyPathCorrect() { + OssStorageResource ossStorageResource = new OssStorageResource(this.oss, + "oss://aliyun-test-bucket/myfilekey", beanFactory, false); + + assertTrue(ossStorageResource.exists()); + } + + @Test + public void testSpecifyBucketCorrect() { + OssStorageResource ossStorageResource = new OssStorageResource(this.oss, + "oss://aliyun-test-bucket", beanFactory, false); + + assertTrue(ossStorageResource.isBucket()); + assertEquals("aliyun-test-bucket", ossStorageResource.getBucket().getName()); + assertTrue(ossStorageResource.exists()); + } + + @Test + public void testBucketOutputStream() throws IOException { + this.expectedEx.expect(IllegalStateException.class); + this.expectedEx.expectMessage( + "Cannot open an output stream to a bucket: 'oss://aliyun-test-bucket/'"); + ((WritableResource) this.bucketResource).getOutputStream(); + } + + @Test + public void testBucketInputStream() throws IOException { + this.expectedEx.expect(IllegalStateException.class); + this.expectedEx.expectMessage( + "Cannot open an input stream to a bucket: 'oss://aliyun-test-bucket/'"); + this.bucketResource.getInputStream(); + } + + @Test + public void testBucketContentLength() throws IOException { + this.expectedEx.expect(FileNotFoundException.class); + this.expectedEx.expectMessage("OSSObject not existed."); + this.bucketResource.contentLength(); + } + + @Test + public void testBucketFile() throws IOException { + this.expectedEx.expect(UnsupportedOperationException.class); + this.expectedEx.expectMessage( + "oss://aliyun-test-bucket/ cannot be resolved to absolute file path"); + this.bucketResource.getFile(); + } + + @Test + public void testBucketLastModified() throws IOException { + this.expectedEx.expect(FileNotFoundException.class); + this.expectedEx.expectMessage("OSSObject not existed."); + this.bucketResource.lastModified(); + } + + @Test + public void testBucketResourceStatuses() { + assertFalse(this.bucketResource.isOpen()); + assertFalse(((WritableResource) this.bucketResource).isWritable()); + assertTrue(this.bucketResource.exists()); + } + + @Test + public void testWritable() throws Exception { + assertTrue(this.remoteResource instanceof WritableResource); + WritableResource writableResource = (WritableResource) this.remoteResource; + assertTrue(writableResource.isWritable()); + writableResource.getOutputStream(); + } + + @Test + public void testWritableOutputStream() throws Exception { + String location = "oss://aliyun-test-bucket/test"; + OssStorageResource resource = new OssStorageResource(this.oss, location, beanFactory,true); + OutputStream os = resource.getOutputStream(); + assertNotNull(os); + + byte[] randomBytes = generateRandomBytes(1203); + String expectedString = new String(randomBytes); + + os.write(randomBytes); + os.close(); + + InputStream in = resource.getInputStream(); + + byte[] result = StreamUtils.copyToByteArray(in); + String actualString = new String(result); + + assertEquals(expectedString, actualString); + } + + @Test + public void testCreateBucket() { + String location = "oss://my-new-test-bucket/"; + OssStorageResource resource = new OssStorageResource(this.oss, location, beanFactory, true); + + resource.createBucket(); + + assertTrue(resource.bucketExists()); + + } + + /** + * Configuration for the tests. + */ + @Configuration + @Import(OssStorageProtocolResolver.class) + static class TestConfiguration { + + @Bean(name = OSS_TASK_EXECUTOR_BEAN_NAME) + @ConditionalOnMissingBean + public ExecutorService ossTaskExecutor() { + return new ThreadPoolExecutor(8, 128, + 60, TimeUnit.SECONDS, new SynchronousQueue<>()); + } + + @Bean + public static OSS mockOSS() { + DummyOssClient dummyOssStub = new DummyOssClient(); + OSS oss = mock(OSS.class); + + doAnswer(invocation -> dummyOssStub.putObject(invocation.getArgument(0), + invocation.getArgument(1), invocation.getArgument(2))).when(oss) + .putObject(Mockito.anyString(), Mockito.anyString(), + Mockito.any(InputStream.class)); + + doAnswer(invocation -> dummyOssStub.getOSSObject(invocation.getArgument(0), + invocation.getArgument(1))).when(oss).getObject(Mockito.anyString(), + Mockito.anyString()); + + doAnswer(invocation -> dummyOssStub.bucketList()).when(oss).listBuckets(); + + doAnswer(invocation -> dummyOssStub.createBucket(invocation.getArgument(0))) + .when(oss).createBucket(Mockito.anyString()); + + // prepare object + dummyOssStub.createBucket("aliyun-test-bucket"); + + byte[] content = generateRandomBytes(4096); + ByteArrayInputStream inputStream = new ByteArrayInputStream(content); + dummyOssStub.putObject("aliyun-test-bucket", "myfilekey", inputStream); + + return oss; + } + + } + +}