Merge pull request #900 from lichen782/master

Support OssStorage Resource as WritableResource and add related unittest cases

spring-cloud-alibaba-examples/oss-example/src/main/java/com/alibaba/cloud/examples/OssController.java

@@ -1,18 +1,20 @@
 package com.alibaba.cloud.examples;
 
-import java.nio.charset.Charset;
-
-import org.apache.commons.codec.CharEncoding;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.core.io.Resource;
-import org.springframework.util.StreamUtils;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.RestController;
-
 import com.aliyun.oss.OSS;
 import com.aliyun.oss.common.utils.IOUtils;
 import com.aliyun.oss.model.OSSObject;
+import org.apache.commons.codec.CharEncoding;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.core.io.Resource;
+import org.springframework.core.io.WritableResource;
+import org.springframework.util.StreamUtils;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
 
 /**
  * OSS Controller
@@ -25,8 +27,11 @@ public class OssController {
 	@Autowired
 	private OSS ossClient;
 
+	@Value("classpath:/oss-test.json")
+	private Resource localFile;
+
 	@Value("oss://" + OssApplication.BUCKET_NAME + "/oss-test.json")
-	private Resource file;
+	private Resource remoteFile;
 
 	@GetMapping("/upload")
 	public String upload() {
@@ -45,7 +50,7 @@ public class OssController {
 	public String fileResource() {
 		try {
 			return "get file resource success. content: " + StreamUtils.copyToString(
-					file.getInputStream(), Charset.forName(CharEncoding.UTF_8));
+					remoteFile.getInputStream(), Charset.forName(CharEncoding.UTF_8));
 		}
 		catch (Exception e) {
 			e.printStackTrace();
@@ -67,4 +72,20 @@ public class OssController {
 		}
 	}
 
+	@GetMapping("/upload2")
+	public String uploadWithOutputStream() {
+		try {
+			try (OutputStream outputStream = ((WritableResource) this.remoteFile)
+					.getOutputStream();
+					InputStream inputStream = localFile.getInputStream()) {
+				StreamUtils.copy(inputStream, outputStream);
+			}
+		}
+		catch (Exception ex) {
+			ex.printStackTrace();
+			return "upload with outputStream failed";
+		}
+		return "upload success";
+	}
+
 }

spring-cloud-alicloud-oss/pom.xml

@@ -58,6 +58,12 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+
     </dependencies>
 
     <build>

spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/OssAutoConfiguration.java

@@ -25,6 +25,11 @@ import org.springframework.context.annotation.Configuration;
 import com.alibaba.alicloud.oss.resource.OssStorageProtocolResolver;
 
 import com.aliyun.oss.OSS;
+import org.springframework.context.annotation.Lazy;
+
+import java.util.concurrent.*;
+
+import static com.alibaba.alicloud.oss.OssConstants.OSS_TASK_EXECUTOR_BEAN_NAME;
 
 /**
  * OSS Auto {@link Configuration}
@@ -42,4 +47,12 @@ public class OssAutoConfiguration {
 		return new OssStorageProtocolResolver();
 	}
 
+	@Bean(name = OSS_TASK_EXECUTOR_BEAN_NAME)
+	@ConditionalOnMissingBean
+	public ExecutorService ossTaskExecutor() {
+		int coreSize = Runtime.getRuntime().availableProcessors();
+		return new ThreadPoolExecutor(coreSize, 128, 60, TimeUnit.SECONDS,
+				new SynchronousQueue<>());
+	}
+
 }

spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/OssConstants.java

@@ -26,4 +26,6 @@ public interface OssConstants {
 	String PREFIX = "spring.cloud.alibaba.oss";
 	String ENABLED = PREFIX + ".enabled";
 
+	String OSS_TASK_EXECUTOR_BEAN_NAME = "ossTaskExecutor";
+
 }

spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageProtocolResolver.java

@@ -63,7 +63,7 @@ public class OssStorageProtocolResolver
 		if (!location.startsWith(PROTOCOL)) {
 			return null;
 		}
-		return new OssStorageResource(getOSS(), location);
+		return new OssStorageResource(getOSS(), location, beanFactory);
 	}
 
 	@Override

spring-cloud-alicloud-oss/src/main/java/com/alibaba/alicloud/oss/resource/OssStorageResource.java

@@ -16,22 +16,25 @@
 
 package com.alibaba.alicloud.oss.resource;
 
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-
-import org.springframework.core.io.Resource;
-import org.springframework.util.Assert;
-
 import com.aliyun.oss.ClientException;
 import com.aliyun.oss.OSS;
 import com.aliyun.oss.OSSException;
 import com.aliyun.oss.model.Bucket;
 import com.aliyun.oss.model.OSSObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.config.ConfigurableListableBeanFactory;
+import org.springframework.core.io.Resource;
+import org.springframework.core.io.WritableResource;
+import org.springframework.util.Assert;
+
+import java.io.*;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.concurrent.ExecutorService;
+
+import static com.alibaba.alicloud.oss.OssConstants.OSS_TASK_EXECUTOR_BEAN_NAME;
 
 /**
  * Implements {@link Resource} for reading and writing objects in Aliyun Object Storage
@@ -43,18 +46,33 @@ import com.aliyun.oss.model.OSSObject;
  * @see Bucket
  * @see OSSObject
  */
-public class OssStorageResource implements Resource {
+public class OssStorageResource implements WritableResource {
+
+	private static final Logger logger = LoggerFactory.getLogger(OssStorageResource.class);
+
+	private static final String MESSAGE_KEY_NOT_EXIST = "The specified key does not exist.";
 
 	private final OSS oss;
 	private final String bucketName;
 	private final String objectKey;
 	private final URI location;
+	private final boolean autoCreateFiles;
 
-	public OssStorageResource(OSS oss, String location) {
+	private final ExecutorService ossTaskExecutor;
+
+	private final ConfigurableListableBeanFactory beanFactory;
+
+	public OssStorageResource(OSS oss, String location, ConfigurableListableBeanFactory beanFactory) {
+		this(oss, location, beanFactory,false);
+    }
+
+	public OssStorageResource(OSS oss, String location,ConfigurableListableBeanFactory beanFactory, boolean autoCreateFiles) {
 		Assert.notNull(oss, "Object Storage Service can not be null");
 		Assert.isTrue(location.startsWith(OssStorageProtocolResolver.PROTOCOL),
 				"Location must start with " + OssStorageProtocolResolver.PROTOCOL);
 		this.oss = oss;
+		this.autoCreateFiles = autoCreateFiles;
+		this.beanFactory = beanFactory;
 		try {
 			URI locationUri = new URI(location);
 			this.bucketName = locationUri.getAuthority();
@@ -70,6 +88,12 @@ public class OssStorageResource implements Resource {
 		catch (URISyntaxException e) {
 			throw new IllegalArgumentException("Invalid location: " + location, e);
 		}
+
+		this.ossTaskExecutor = this.beanFactory.getBean(OSS_TASK_EXECUTOR_BEAN_NAME, ExecutorService.class);
+	}
+
+	public boolean isAutoCreateFiles() {
+		return this.autoCreateFiles;
 	}
 
 	@Override
@@ -125,7 +149,7 @@ public class OssStorageResource implements Resource {
 	@Override
 	public Resource createRelative(String relativePath) throws IOException {
 		return new OssStorageResource(this.oss,
-				this.location.resolve(relativePath).toString());
+				this.location.resolve(relativePath).toString(), this.beanFactory);
 	}
 
 	@Override
@@ -193,4 +217,70 @@ public class OssStorageResource implements Resource {
 		}
 	}
 
+	/**
+	 * create a bucket.
+	 * @return
+	 */
+	public Bucket createBucket() {
+		return this.oss.createBucket(this.bucketName);
+	}
+
+	@Override
+	public boolean isWritable() {
+		return !isBucket() && (this.autoCreateFiles || exists());
+	}
+
+	/**
+	 * acquire an OutputStream for write.
+	 * Note: please close the stream after writing is done
+	 * @return
+	 * @throws IOException
+	 */
+	@Override
+	public OutputStream getOutputStream() throws IOException {
+		if (isBucket()) {
+			throw new IllegalStateException(
+					"Cannot open an output stream to a bucket: '" + getURI() + "'");
+		}
+		else {
+			OSSObject ossObject;
+
+			try {
+				ossObject = this.getOSSObject();
+			}
+			catch (OSSException ex) {
+				if (ex.getMessage() != null
+						&& ex.getMessage().startsWith(MESSAGE_KEY_NOT_EXIST)) {
+					ossObject = null;
+				}
+				else {
+					throw ex;
+				}
+			}
+
+			if (ossObject == null) {
+				if (!this.autoCreateFiles) {
+					throw new FileNotFoundException(
+							"The object was not found: " + getURI());
+				}
+
+			}
+
+			PipedInputStream in = new PipedInputStream();
+			final PipedOutputStream out = new PipedOutputStream(in);
+
+			ossTaskExecutor.submit(() -> {
+				try {
+					OssStorageResource.this.oss.putObject(bucketName, objectKey, in);
+				}
+				catch (Exception ex) {
+					logger.error("Failed to put object", ex);
+				}
+			});
+
+			return out;
+		}
+
+	}
+
 }

spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/DummyOssClient.java

@@ -0,0 +1,98 @@
+/*
+ * Copyright (C) 2018 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.alibaba.alicloud.oss.resource;
+
+import com.aliyun.oss.model.Bucket;
+import com.aliyun.oss.model.OSSObject;
+import com.aliyun.oss.model.ObjectMetadata;
+import com.aliyun.oss.model.PutObjectResult;
+import org.springframework.util.StreamUtils;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * @author lich
+ */
+public class DummyOssClient {
+
+	private Map<String, byte[]> storeMap = new ConcurrentHashMap<>();
+
+	private Map<String, Bucket> bucketSet = new HashMap<>();
+
+	public String getStoreKey(String bucketName, String objectKey) {
+		return String.join(".", bucketName, objectKey);
+	}
+
+	public PutObjectResult putObject(String bucketName, String objectKey,
+			InputStream inputStream) {
+
+		try {
+			byte[] result = StreamUtils.copyToByteArray(inputStream);
+			storeMap.put(getStoreKey(bucketName, objectKey), result);
+		}
+		catch (IOException ex) {
+			throw new RuntimeException(ex);
+		}
+		finally {
+			try {
+				inputStream.close();
+			}
+			catch (IOException ex) {
+				throw new RuntimeException(ex);
+			}
+		}
+
+		return new PutObjectResult();
+	}
+
+	public OSSObject getOSSObject(String bucketName, String objectKey) {
+		byte[] value = storeMap.get(this.getStoreKey(bucketName, objectKey));
+		if (value == null) {
+			return null;
+		}
+		OSSObject ossObject = new OSSObject();
+		ossObject.setBucketName(bucketName);
+		ossObject.setKey(objectKey);
+		InputStream inputStream = new ByteArrayInputStream(value);
+		ossObject.setObjectContent(inputStream);
+
+		ObjectMetadata objectMetadata = new ObjectMetadata();
+		objectMetadata.setContentLength(value.length);
+		ossObject.setObjectMetadata(objectMetadata);
+
+		return ossObject;
+	}
+
+	public Bucket createBucket(String bucketName) {
+		if (bucketSet.containsKey(bucketName)) {
+			return bucketSet.get(bucketName);
+		}
+		Bucket bucket = new Bucket();
+		bucket.setCreationDate(new Date());
+		bucket.setName(bucketName);
+		bucketSet.put(bucketName, bucket);
+		return bucket;
+	}
+
+	public List<Bucket> bucketList() {
+		return new ArrayList<>(bucketSet.values());
+	}
+}

spring-cloud-alicloud-oss/src/test/java/com/alibaba/alicloud/oss/resource/OssStorageResourceTest.java

@@ -0,0 +1,266 @@
+/*
+ * Copyright (C) 2018 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.alibaba.alicloud.oss.resource;
+
+import com.aliyun.oss.OSS;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.beans.factory.config.BeanFactoryPostProcessor;
+import org.springframework.beans.factory.config.ConfigurableListableBeanFactory;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+import org.springframework.core.io.Resource;
+import org.springframework.core.io.WritableResource;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.util.StreamUtils;
+
+import java.io.*;
+import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import static com.alibaba.alicloud.oss.OssConstants.OSS_TASK_EXECUTOR_BEAN_NAME;
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+
+/**
+ * @author lich
+ */
+@SpringBootTest
+@RunWith(SpringRunner.class)
+public class OssStorageResourceTest {
+
+	/**
+	 * Used to test exception messages and types.
+	 */
+	@Rule
+	public ExpectedException expectedEx = ExpectedException.none();
+
+	@Autowired
+	private ConfigurableListableBeanFactory beanFactory;
+
+	@Autowired
+	private OSS oss;
+
+	@Value("oss://aliyun-test-bucket/")
+	private Resource bucketResource;
+
+	@Value("oss://aliyun-test-bucket/myfilekey")
+	private Resource remoteResource;
+
+	public static byte[] generateRandomBytes(int blen) {
+		byte[] array = new byte[blen];
+		new Random().nextBytes(array);
+		return array;
+	}
+
+	@Test
+	public void testResourceType() {
+		assertEquals(OssStorageResource.class, remoteResource.getClass());
+		OssStorageResource ossStorageResource = (OssStorageResource) remoteResource;
+		assertEquals("myfilekey", ossStorageResource.getFilename());
+		assertFalse(ossStorageResource.isBucket());
+	}
+
+	@Test
+	public void testValidObject() throws Exception {
+		assertTrue(remoteResource.exists());
+		OssStorageResource ossStorageResource = (OssStorageResource) remoteResource;
+		assertTrue(ossStorageResource.bucketExists());
+		assertEquals(4096L, remoteResource.contentLength());
+		assertEquals("oss://aliyun-test-bucket/myfilekey",
+				remoteResource.getURI().toString());
+		assertEquals("myfilekey", remoteResource.getFilename());
+	}
+
+	@Test
+	public void testBucketResource() throws Exception {
+		assertTrue(bucketResource.exists());
+		assertTrue(((OssStorageResource) this.bucketResource).isBucket());
+		assertTrue(((OssStorageResource) this.bucketResource).bucketExists());
+		assertEquals("oss://aliyun-test-bucket/", bucketResource.getURI().toString());
+		assertEquals("aliyun-test-bucket", this.bucketResource.getFilename());
+	}
+
+	@Test
+	public void testBucketNotEndingInSlash() {
+		assertTrue(
+				new OssStorageResource(this.oss, "oss://aliyun-test-bucket", beanFactory)
+						.isBucket());
+	}
+
+	@Test
+	public void testSpecifyPathCorrect() {
+		OssStorageResource ossStorageResource = new OssStorageResource(this.oss,
+				"oss://aliyun-test-bucket/myfilekey", beanFactory, false);
+
+		assertTrue(ossStorageResource.exists());
+	}
+
+	@Test
+	public void testSpecifyBucketCorrect() {
+		OssStorageResource ossStorageResource = new OssStorageResource(this.oss,
+				"oss://aliyun-test-bucket", beanFactory, false);
+
+		assertTrue(ossStorageResource.isBucket());
+		assertEquals("aliyun-test-bucket", ossStorageResource.getBucket().getName());
+		assertTrue(ossStorageResource.exists());
+	}
+
+	@Test
+	public void testBucketOutputStream() throws IOException {
+		this.expectedEx.expect(IllegalStateException.class);
+		this.expectedEx.expectMessage(
+				"Cannot open an output stream to a bucket: 'oss://aliyun-test-bucket/'");
+		((WritableResource) this.bucketResource).getOutputStream();
+	}
+
+	@Test
+	public void testBucketInputStream() throws IOException {
+		this.expectedEx.expect(IllegalStateException.class);
+		this.expectedEx.expectMessage(
+				"Cannot open an input stream to a bucket: 'oss://aliyun-test-bucket/'");
+		this.bucketResource.getInputStream();
+	}
+
+	@Test
+	public void testBucketContentLength() throws IOException {
+		this.expectedEx.expect(FileNotFoundException.class);
+		this.expectedEx.expectMessage("OSSObject not existed.");
+		this.bucketResource.contentLength();
+	}
+
+	@Test
+	public void testBucketFile() throws IOException {
+		this.expectedEx.expect(UnsupportedOperationException.class);
+		this.expectedEx.expectMessage(
+				"oss://aliyun-test-bucket/ cannot be resolved to absolute file path");
+		this.bucketResource.getFile();
+	}
+
+	@Test
+	public void testBucketLastModified() throws IOException {
+		this.expectedEx.expect(FileNotFoundException.class);
+		this.expectedEx.expectMessage("OSSObject not existed.");
+		this.bucketResource.lastModified();
+	}
+
+	@Test
+	public void testBucketResourceStatuses() {
+		assertFalse(this.bucketResource.isOpen());
+		assertFalse(((WritableResource) this.bucketResource).isWritable());
+		assertTrue(this.bucketResource.exists());
+	}
+
+	@Test
+	public void testWritable() throws Exception {
+		assertTrue(this.remoteResource instanceof WritableResource);
+		WritableResource writableResource = (WritableResource) this.remoteResource;
+		assertTrue(writableResource.isWritable());
+		writableResource.getOutputStream();
+	}
+
+	@Test
+	public void testWritableOutputStream() throws Exception {
+		String location = "oss://aliyun-test-bucket/test";
+		OssStorageResource resource = new OssStorageResource(this.oss, location, beanFactory,true);
+		OutputStream os = resource.getOutputStream();
+		assertNotNull(os);
+
+		byte[] randomBytes = generateRandomBytes(1203);
+		String expectedString = new String(randomBytes);
+
+		os.write(randomBytes);
+		os.close();
+
+		InputStream in = resource.getInputStream();
+
+		byte[] result = StreamUtils.copyToByteArray(in);
+		String actualString = new String(result);
+
+		assertEquals(expectedString, actualString);
+	}
+
+	@Test
+	public void testCreateBucket() {
+		String location = "oss://my-new-test-bucket/";
+		OssStorageResource resource = new OssStorageResource(this.oss, location, beanFactory, true);
+
+		resource.createBucket();
+
+		assertTrue(resource.bucketExists());
+
+	}
+
+	/**
+	 * Configuration for the tests.
+	 */
+	@Configuration
+	@Import(OssStorageProtocolResolver.class)
+	static class TestConfiguration {
+
+		@Bean(name = OSS_TASK_EXECUTOR_BEAN_NAME)
+		@ConditionalOnMissingBean
+		public ExecutorService ossTaskExecutor() {
+			return new ThreadPoolExecutor(8, 128,
+				60, TimeUnit.SECONDS, new SynchronousQueue<>());
+		}
+
+		@Bean
+		public static OSS mockOSS() {
+			DummyOssClient dummyOssStub = new DummyOssClient();
+			OSS oss = mock(OSS.class);
+
+			doAnswer(invocation -> dummyOssStub.putObject(invocation.getArgument(0),
+					invocation.getArgument(1), invocation.getArgument(2))).when(oss)
+							.putObject(Mockito.anyString(), Mockito.anyString(),
+									Mockito.any(InputStream.class));
+
+			doAnswer(invocation -> dummyOssStub.getOSSObject(invocation.getArgument(0),
+					invocation.getArgument(1))).when(oss).getObject(Mockito.anyString(),
+							Mockito.anyString());
+
+			doAnswer(invocation -> dummyOssStub.bucketList()).when(oss).listBuckets();
+
+			doAnswer(invocation -> dummyOssStub.createBucket(invocation.getArgument(0)))
+					.when(oss).createBucket(Mockito.anyString());
+
+			// prepare object
+			dummyOssStub.createBucket("aliyun-test-bucket");
+
+			byte[] content = generateRandomBytes(4096);
+			ByteArrayInputStream inputStream = new ByteArrayInputStream(content);
+			dummyOssStub.putObject("aliyun-test-bucket", "myfilekey", inputStream);
+
+			return oss;
+		}
+
+	}
+
+}