1
0
mirror of https://gitee.com/mirrors/Spring-Cloud-Alibaba.git synced 2021-06-26 13:25:11 +08:00

sync & commit in finchley

This commit is contained in:
fangjian0423
2019-10-30 13:10:10 +08:00
parent 6be45914c8
commit 15465b5612
436 changed files with 7099 additions and 3202 deletions

View File

@@ -58,5 +58,11 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

View File

@@ -18,13 +18,13 @@ package com.alibaba.alicloud.oss;
import java.util.Map;
import com.aliyun.oss.OSS;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationListener;
import org.springframework.context.event.ContextClosedEvent;
import com.aliyun.oss.OSS;
/**
* Shutdown All OSS Clients when {@code ApplicationContext} gets closed
* {@link ApplicationListener}

View File

@@ -16,18 +16,24 @@
package com.alibaba.alicloud.oss;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import com.alibaba.alicloud.oss.resource.OssStorageProtocolResolver;
import com.aliyun.oss.OSS;
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import com.alibaba.alicloud.oss.resource.OssStorageProtocolResolver;
import com.aliyun.oss.OSS;
import static com.alibaba.alicloud.oss.OssConstants.OSS_TASK_EXECUTOR_BEAN_NAME;
/**
* OSS Auto {@link Configuration}
* OSS Auto {@link Configuration}.
*
* @author <a href="mailto:fangjian0423@gmail.com">Jim</a>
*/
@@ -42,4 +48,12 @@ public class OssAutoConfiguration {
return new OssStorageProtocolResolver();
}
@Bean(name = OSS_TASK_EXECUTOR_BEAN_NAME)
@ConditionalOnMissingBean
public ExecutorService ossTaskExecutor() {
int coreSize = Runtime.getRuntime().availableProcessors();
return new ThreadPoolExecutor(coreSize, 128, 60, TimeUnit.SECONDS,
new SynchronousQueue<>());
}
}

View File

@@ -17,13 +17,29 @@
package com.alibaba.alicloud.oss;
/**
* OSS constants
* OSS constants.
*
* @author <a href="mailto:fangjian0423@gmail.com">Jim</a>
*/
public interface OssConstants {
public final class OssConstants {
String PREFIX = "spring.cloud.alibaba.oss";
String ENABLED = PREFIX + ".enabled";
/**
* Prefix of OSSConfigurationProperties.
*/
public static final String PREFIX = "spring.cloud.alibaba.oss";
/**
* Enable OSS.
*/
public static final String ENABLED = PREFIX + ".enabled";
/**
* OSS ThreadPool bean name.
*/
public static final String OSS_TASK_EXECUTOR_BEAN_NAME = "ossTaskExecutor";
private OssConstants() {
throw new AssertionError("Must not instantiate constant utility class");
}
}

View File

@@ -21,13 +21,13 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.aliyun.oss.OSSClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.actuate.endpoint.annotation.Endpoint;
import org.springframework.boot.actuate.endpoint.annotation.ReadOperation;
import org.springframework.context.ApplicationContext;
import com.aliyun.oss.OSSClient;
/**
* Actuator {@link Endpoint} to expose OSS Meta Data
*

View File

@@ -16,8 +16,10 @@
package com.alibaba.alicloud.oss.resource;
import com.aliyun.oss.OSS;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.config.BeanFactoryPostProcessor;
import org.springframework.beans.factory.config.ConfigurableListableBeanFactory;
@@ -27,8 +29,6 @@ import org.springframework.core.io.ProtocolResolver;
import org.springframework.core.io.Resource;
import org.springframework.core.io.ResourceLoader;
import com.aliyun.oss.OSS;
/**
* A {@link ProtocolResolver} implementation for the {@code oss://} protocol.
*
@@ -37,6 +37,9 @@ import com.aliyun.oss.OSS;
public class OssStorageProtocolResolver
implements ProtocolResolver, BeanFactoryPostProcessor, ResourceLoaderAware {
/**
* protocol of oss resource.
*/
public static final String PROTOCOL = "oss://";
private static final Logger log = LoggerFactory
@@ -63,7 +66,7 @@ public class OssStorageProtocolResolver
if (!location.startsWith(PROTOCOL)) {
return null;
}
return new OssStorageResource(getOSS(), location);
return new OssStorageResource(getOSS(), location, beanFactory);
}
@Override
@@ -82,4 +85,5 @@ public class OssStorageProtocolResolver
throws BeansException {
this.beanFactory = beanFactory;
}
}

View File

@@ -20,18 +20,28 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import org.springframework.core.io.Resource;
import org.springframework.util.Assert;
import java.util.concurrent.ExecutorService;
import com.aliyun.oss.ClientException;
import com.aliyun.oss.OSS;
import com.aliyun.oss.OSSException;
import com.aliyun.oss.model.Bucket;
import com.aliyun.oss.model.OSSObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.config.ConfigurableListableBeanFactory;
import org.springframework.core.io.Resource;
import org.springframework.core.io.WritableResource;
import org.springframework.util.Assert;
import static com.alibaba.alicloud.oss.OssConstants.OSS_TASK_EXECUTOR_BEAN_NAME;
/**
* Implements {@link Resource} for reading and writing objects in Aliyun Object Storage
@@ -43,18 +53,40 @@ import com.aliyun.oss.model.OSSObject;
* @see Bucket
* @see OSSObject
*/
public class OssStorageResource implements Resource {
public class OssStorageResource implements WritableResource {
private static final Logger logger = LoggerFactory
.getLogger(OssStorageResource.class);
private static final String MESSAGE_KEY_NOT_EXIST = "The specified key does not exist.";
private final OSS oss;
private final String bucketName;
private final String objectKey;
private final URI location;
public OssStorageResource(OSS oss, String location) {
private final boolean autoCreateFiles;
private final ExecutorService ossTaskExecutor;
private final ConfigurableListableBeanFactory beanFactory;
public OssStorageResource(OSS oss, String location,
ConfigurableListableBeanFactory beanFactory) {
this(oss, location, beanFactory, false);
}
public OssStorageResource(OSS oss, String location,
ConfigurableListableBeanFactory beanFactory, boolean autoCreateFiles) {
Assert.notNull(oss, "Object Storage Service can not be null");
Assert.isTrue(location.startsWith(OssStorageProtocolResolver.PROTOCOL),
"Location must start with " + OssStorageProtocolResolver.PROTOCOL);
this.oss = oss;
this.autoCreateFiles = autoCreateFiles;
this.beanFactory = beanFactory;
try {
URI locationUri = new URI(location);
this.bucketName = locationUri.getAuthority();
@@ -70,6 +102,13 @@ public class OssStorageResource implements Resource {
catch (URISyntaxException e) {
throw new IllegalArgumentException("Invalid location: " + location, e);
}
this.ossTaskExecutor = this.beanFactory.getBean(OSS_TASK_EXECUTOR_BEAN_NAME,
ExecutorService.class);
}
public boolean isAutoCreateFiles() {
return this.autoCreateFiles;
}
@Override
@@ -125,7 +164,7 @@ public class OssStorageResource implements Resource {
@Override
public Resource createRelative(String relativePath) throws IOException {
return new OssStorageResource(this.oss,
this.location.resolve(relativePath).toString());
this.location.resolve(relativePath).toString(), this.beanFactory);
}
@Override
@@ -157,7 +196,7 @@ public class OssStorageResource implements Resource {
public Bucket getBucket() {
return this.oss.listBuckets().stream()
.filter(bucket -> bucket.getName().equals(this.bucketName)).findFirst()
.get();
.orElse(null);
}
/**
@@ -193,4 +232,70 @@ public class OssStorageResource implements Resource {
}
}
/**
* create a bucket.
* @return OSS Bucket
*/
public Bucket createBucket() {
return this.oss.createBucket(this.bucketName);
}
@Override
public boolean isWritable() {
return !isBucket() && (this.autoCreateFiles || exists());
}
/**
* acquire an OutputStream for write. Note: please close the stream after writing is
* done
* @return OutputStream of OSS resource
* @throws IOException throw by oss operation
*/
@Override
public OutputStream getOutputStream() throws IOException {
if (isBucket()) {
throw new IllegalStateException(
"Cannot open an output stream to a bucket: '" + getURI() + "'");
}
else {
OSSObject ossObject;
try {
ossObject = this.getOSSObject();
}
catch (OSSException ex) {
if (ex.getMessage() != null
&& ex.getMessage().startsWith(MESSAGE_KEY_NOT_EXIST)) {
ossObject = null;
}
else {
throw ex;
}
}
if (ossObject == null) {
if (!this.autoCreateFiles) {
throw new FileNotFoundException(
"The object was not found: " + getURI());
}
}
PipedInputStream in = new PipedInputStream();
final PipedOutputStream out = new PipedOutputStream(in);
ossTaskExecutor.submit(() -> {
try {
OssStorageResource.this.oss.putObject(bucketName, objectKey, in);
}
catch (Exception ex) {
logger.error("Failed to put object", ex);
}
});
return out;
}
}
}

View File

@@ -0,0 +1,104 @@
/*
* Copyright 2013-2018 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.alicloud.oss.resource;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import com.aliyun.oss.model.Bucket;
import com.aliyun.oss.model.OSSObject;
import com.aliyun.oss.model.ObjectMetadata;
import com.aliyun.oss.model.PutObjectResult;
import org.springframework.util.StreamUtils;
/**
* @author lich
*/
public class DummyOssClient {
private Map<String, byte[]> storeMap = new ConcurrentHashMap<>();
private Map<String, Bucket> bucketSet = new HashMap<>();
public String getStoreKey(String bucketName, String objectKey) {
return String.join(".", bucketName, objectKey);
}
public PutObjectResult putObject(String bucketName, String objectKey,
InputStream inputStream) {
try {
byte[] result = StreamUtils.copyToByteArray(inputStream);
storeMap.put(getStoreKey(bucketName, objectKey), result);
}
catch (IOException ex) {
throw new RuntimeException(ex);
}
finally {
try {
inputStream.close();
}
catch (IOException ex) {
throw new RuntimeException(ex);
}
}
return new PutObjectResult();
}
public OSSObject getOSSObject(String bucketName, String objectKey) {
byte[] value = storeMap.get(this.getStoreKey(bucketName, objectKey));
if (value == null) {
return null;
}
OSSObject ossObject = new OSSObject();
ossObject.setBucketName(bucketName);
ossObject.setKey(objectKey);
InputStream inputStream = new ByteArrayInputStream(value);
ossObject.setObjectContent(inputStream);
ObjectMetadata objectMetadata = new ObjectMetadata();
objectMetadata.setContentLength(value.length);
ossObject.setObjectMetadata(objectMetadata);
return ossObject;
}
public Bucket createBucket(String bucketName) {
if (bucketSet.containsKey(bucketName)) {
return bucketSet.get(bucketName);
}
Bucket bucket = new Bucket();
bucket.setCreationDate(new Date());
bucket.setName(bucketName);
bucketSet.put(bucketName, bucket);
return bucket;
}
public List<Bucket> bucketList() {
return new ArrayList<>(bucketSet.values());
}
}

View File

@@ -0,0 +1,273 @@
/*
* Copyright 2013-2018 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.alicloud.oss.resource;
import java.io.ByteArrayInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Random;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import com.aliyun.oss.OSS;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.config.ConfigurableListableBeanFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.core.io.Resource;
import org.springframework.core.io.WritableResource;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.util.StreamUtils;
import static com.alibaba.alicloud.oss.OssConstants.OSS_TASK_EXECUTOR_BEAN_NAME;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
/**
* @author lich
*/
@SpringBootTest
@RunWith(SpringRunner.class)
public class OssStorageResourceTest {
/**
* Used to test exception messages and types.
*/
@Rule
public ExpectedException expectedEx = ExpectedException.none();
@Autowired
private ConfigurableListableBeanFactory beanFactory;
@Autowired
private OSS oss;
@Value("oss://aliyun-test-bucket/")
private Resource bucketResource;
@Value("oss://aliyun-test-bucket/myfilekey")
private Resource remoteResource;
public static byte[] generateRandomBytes(int blen) {
byte[] array = new byte[blen];
new Random().nextBytes(array);
return array;
}
@Test
public void testResourceType() {
assertThat(remoteResource.getClass()).isEqualTo(OssStorageResource.class);
OssStorageResource ossStorageResource = (OssStorageResource) remoteResource;
assertThat(ossStorageResource.getFilename()).isEqualTo("myfilekey");
assertThat(ossStorageResource.isBucket()).isEqualTo(false);
}
@Test
public void testValidObject() throws Exception {
assertThat(remoteResource.exists()).isEqualTo(true);
OssStorageResource ossStorageResource = (OssStorageResource) remoteResource;
assertThat(ossStorageResource.bucketExists()).isEqualTo(true);
assertThat(remoteResource.contentLength()).isEqualTo(4096L);
assertThat(remoteResource.getURI().toString())
.isEqualTo("oss://aliyun-test-bucket/myfilekey");
assertThat(remoteResource.getFilename()).isEqualTo("myfilekey");
}
@Test
public void testBucketResource() throws Exception {
assertThat(bucketResource.exists()).isEqualTo(true);
assertThat(((OssStorageResource) this.bucketResource).isBucket()).isEqualTo(true);
assertThat(((OssStorageResource) this.bucketResource).bucketExists())
.isEqualTo(true);
assertThat(bucketResource.getURI().toString())
.isEqualTo("oss://aliyun-test-bucket/");
assertThat(this.bucketResource.getFilename()).isEqualTo("aliyun-test-bucket");
}
@Test
public void testBucketNotEndingInSlash() {
assertThat(
new OssStorageResource(this.oss, "oss://aliyun-test-bucket", beanFactory)
.isBucket()).isEqualTo(true);
}
@Test
public void testSpecifyPathCorrect() {
OssStorageResource ossStorageResource = new OssStorageResource(this.oss,
"oss://aliyun-test-bucket/myfilekey", beanFactory, false);
assertThat(ossStorageResource.exists()).isEqualTo(true);
}
@Test
public void testSpecifyBucketCorrect() {
OssStorageResource ossStorageResource = new OssStorageResource(this.oss,
"oss://aliyun-test-bucket", beanFactory, false);
assertThat(ossStorageResource.isBucket()).isEqualTo(true);
assertThat(ossStorageResource.getBucket().getName())
.isEqualTo("aliyun-test-bucket");
assertThat(ossStorageResource.exists()).isEqualTo(true);
}
@Test
public void testBucketOutputStream() throws IOException {
this.expectedEx.expect(IllegalStateException.class);
this.expectedEx.expectMessage(
"Cannot open an output stream to a bucket: 'oss://aliyun-test-bucket/'");
((WritableResource) this.bucketResource).getOutputStream();
}
@Test
public void testBucketInputStream() throws IOException {
this.expectedEx.expect(IllegalStateException.class);
this.expectedEx.expectMessage(
"Cannot open an input stream to a bucket: 'oss://aliyun-test-bucket/'");
this.bucketResource.getInputStream();
}
@Test
public void testBucketContentLength() throws IOException {
this.expectedEx.expect(FileNotFoundException.class);
this.expectedEx.expectMessage("OSSObject not existed.");
this.bucketResource.contentLength();
}
@Test
public void testBucketFile() throws IOException {
this.expectedEx.expect(UnsupportedOperationException.class);
this.expectedEx.expectMessage(
"oss://aliyun-test-bucket/ cannot be resolved to absolute file path");
this.bucketResource.getFile();
}
@Test
public void testBucketLastModified() throws IOException {
this.expectedEx.expect(FileNotFoundException.class);
this.expectedEx.expectMessage("OSSObject not existed.");
this.bucketResource.lastModified();
}
@Test
public void testBucketResourceStatuses() {
assertThat(this.bucketResource.isOpen()).isEqualTo(false);
assertThat(((WritableResource) this.bucketResource).isWritable())
.isEqualTo(false);
assertThat(this.bucketResource.exists()).isEqualTo(true);
}
@Test
public void testWritable() throws Exception {
assertThat(this.remoteResource instanceof WritableResource).isEqualTo(true);
WritableResource writableResource = (WritableResource) this.remoteResource;
assertThat(writableResource.isWritable()).isEqualTo(true);
writableResource.getOutputStream();
}
@Test
public void testWritableOutputStream() throws Exception {
String location = "oss://aliyun-test-bucket/test";
OssStorageResource resource = new OssStorageResource(this.oss, location,
beanFactory, true);
OutputStream os = resource.getOutputStream();
assertThat(os).isNotNull();
byte[] randomBytes = generateRandomBytes(1203);
String expectedString = new String(randomBytes);
os.write(randomBytes);
os.close();
InputStream in = resource.getInputStream();
byte[] result = StreamUtils.copyToByteArray(in);
String actualString = new String(result);
assertThat(actualString).isEqualTo(expectedString);
}
@Test
public void testCreateBucket() {
String location = "oss://my-new-test-bucket/";
OssStorageResource resource = new OssStorageResource(this.oss, location,
beanFactory, true);
resource.createBucket();
assertThat(resource.bucketExists()).isEqualTo(true);
}
/**
* Configuration for the tests.
*/
@Configuration
@Import(OssStorageProtocolResolver.class)
static class TestConfiguration {
@Bean(name = OSS_TASK_EXECUTOR_BEAN_NAME)
@ConditionalOnMissingBean
public ExecutorService ossTaskExecutor() {
return new ThreadPoolExecutor(8, 128, 60, TimeUnit.SECONDS,
new SynchronousQueue<>());
}
@Bean
public static OSS mockOSS() {
DummyOssClient dummyOssStub = new DummyOssClient();
OSS oss = mock(OSS.class);
doAnswer(invocation -> dummyOssStub.putObject(invocation.getArgument(0),
invocation.getArgument(1), invocation.getArgument(2))).when(oss)
.putObject(Mockito.anyString(), Mockito.anyString(),
Mockito.any(InputStream.class));
doAnswer(invocation -> dummyOssStub.getOSSObject(invocation.getArgument(0),
invocation.getArgument(1))).when(oss).getObject(Mockito.anyString(),
Mockito.anyString());
doAnswer(invocation -> dummyOssStub.bucketList()).when(oss).listBuckets();
doAnswer(invocation -> dummyOssStub.createBucket(invocation.getArgument(0)))
.when(oss).createBucket(Mockito.anyString());
// prepare object
dummyOssStub.createBucket("aliyun-test-bucket");
byte[] content = generateRandomBytes(4096);
ByteArrayInputStream inputStream = new ByteArrayInputStream(content);
dummyOssStub.putObject("aliyun-test-bucket", "myfilekey", inputStream);
return oss;
}
}
}