Merge remote-tracking branch 'origin/master'

pom.xml

@@ -8,7 +8,7 @@
     <artifactId>yexuejc-base</artifactId>
     <version>1.5.0-jre8</version>
 
-    <name>${project.artifactId}</name>
+    <name>yexuejc-base</name>
     <url>https://github.com/yexuejc/yexuejc-base</url>
     <description>Common toolkits based on JDK8 packaging</description>
 
@@ -40,7 +40,7 @@
 
     <properties>
         <repos.yexuejc.url>https://nexus.yexuejc.top/repository/</repos.yexuejc.url>
-        <repos.aliyun.url>http://maven.aliyun.com/nexus/content/groups/public</repos.aliyun.url>
+        <repos.aliyun.url>https://maven.aliyun.com/repository/public</repos.aliyun.url>
         <repos.jitpack.url>https://jitpack.io</repos.jitpack.url>
         <jjwt.version>0.11.5</jjwt.version>
         <maven.compiler.verbose>true</maven.compiler.verbose>
@@ -51,6 +51,7 @@
         <bcprov-jdk15on.version>1.70</bcprov-jdk15on.version>
         <guava.version>31.1-jre</guava.version>
         <apache-poi.version>5.2.2</apache-poi.version>
+        <jackson.version>2.13.2</jackson.version>
         <!-- 文件拷贝时的编码 -->
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
@@ -111,6 +112,12 @@
             <artifactId>poi-ooxml</artifactId>
             <version>${apache-poi.version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-csv</artifactId>
+            <version>${jackson.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter-api</artifactId>
@@ -196,12 +203,12 @@
     </build>
     <repositories>
         <repository>
-            <id>yexuejc-nexus-public</id>
+            <id>yexuejc-maven</id>
             <name>yexuejc-nexus-public</name>
             <url>${repos.yexuejc.url}maven-public/</url>
         </repository>
         <repository>
-            <id>aliyun-nexus-public</id>
+            <id>aliyun-maven</id>
             <name>aliyun-nexus-public</name>
             <url>${repos.aliyun.url}</url>
         </repository>

src/main/java/com/yexuejc/base/annotation/CsvHeader.java

@@ -0,0 +1,21 @@
+package com.yexuejc.base.annotation;
+
+import java.lang.annotation.*;
+
+/**
+ * 设置csv header
+ *
+ * @author MAXF-MAC
+ */
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Documented
+@Inherited
+public @interface CsvHeader {
+
+    /**
+     * 在类头上设置csv格式的header
+     * @return
+     */
+    String header();
+}

src/main/java/com/yexuejc/base/annotation/ToUeProperty.java

@@ -1,4 +1,6 @@
-package com.yexuejc.base.util;
+package com.yexuejc.base.annotation;
+
+import com.yexuejc.base.util.ObjUtil;
 
 import java.lang.annotation.*;
 

src/main/java/com/yexuejc/base/pojo/ReadFileBean.java

@@ -0,0 +1,85 @@
+package com.yexuejc.base.pojo;
+
+import java.util.List;
+
+/**
+ * @author maxf
+ * @class-name ReadFileBean
+ * @description 分段读取大文件
+ * @date 2022/5/16 21:53
+ */
+public class ReadFileBean<T> {
+    /**
+     * 开始行数
+     */
+    private int startRowNum;
+    /**
+     * 结束行数
+     */
+    private int endRowNum;
+
+    /**
+     * 每次读取的行数
+     */
+    private int readRowNum;
+    /**
+     * 开始行到结束行的数据
+     */
+    private List<T> datas;
+    /**
+     * 文件指针位置,默认0，开始位置
+     */
+    private long pointer = 0;
+    /**
+     * 文件的length
+     */
+    private long fileLength;
+
+
+    public ReadFileBean(int readRow) {
+        this.readRowNum = readRow;
+        this.startRowNum = 1;
+        this.endRowNum = 0;
+    }
+
+    public int getReadRowNum() {
+        return readRowNum;
+    }
+
+    public int getStartRowNum() {
+        return startRowNum;
+    }
+
+    public int getEndRowNum() {
+        return endRowNum;
+    }
+
+    public List<T> getDatas() {
+        return datas;
+    }
+
+    public ReadFileBean<T> setDatas(List<T> datas) {
+        this.datas = datas;
+        this.startRowNum = this.endRowNum + 1;
+        this.endRowNum += this.readRowNum;
+        return this;
+    }
+
+    public long getPointer() {
+        return pointer;
+    }
+
+    public ReadFileBean<T> setPointer(long pointer) {
+        this.pointer = pointer;
+        return this;
+    }
+
+    public long getFileLength() {
+        return fileLength;
+    }
+
+    public ReadFileBean<T> setFileLength(long fileLength) {
+        this.fileLength = fileLength;
+        return this;
+    }
+}

src/main/java/com/yexuejc/base/util/FileUtil.java

@@ -1,13 +1,29 @@
 package com.yexuejc.base.util;
 
+import com.fasterxml.jackson.databind.MappingIterator;
+import com.fasterxml.jackson.dataformat.csv.CsvMapper;
+import com.fasterxml.jackson.dataformat.csv.CsvSchema;
+import com.google.common.io.ByteStreams;
+import com.google.common.io.CharStreams;
+import com.yexuejc.base.annotation.CsvHeader;
+import com.yexuejc.base.pojo.ReadFileBean;
+import io.jsonwebtoken.lang.Assert;
+import org.apache.commons.io.IOUtils;
+
 import java.io.*;
 import java.math.BigInteger;
 import java.nio.MappedByteBuffer;
 import java.nio.channels.FileChannel;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
 import java.util.Base64;
+import java.util.List;
+import java.util.Scanner;
 import java.util.logging.Logger;
+import java.util.stream.Collectors;
 import java.util.zip.CRC32;
 
 /**
@@ -25,6 +41,7 @@ public class FileUtil {
     }
 
     private static final String TYPE_TAR_GZ = ".tar.gz";
+    private static final String TYPE_CSV = ".csv";
     private static final String TAR_GZ = "tar.gz";
 
     /**
@@ -49,7 +66,7 @@ public class FileUtil {
     }
 
     /**
-     * 判断文件是否存在
+     * 判断文件是否存在,不存在就创建一个空的
      *
      * @param file
      */
@@ -405,6 +422,325 @@ public class FileUtil {
         return 0;
     }
 
+    /**
+     * 字符串（csv格式）转 对象
+     *
+     * @param data      转换的字符串 如
+     *                  <p> ------------ </p>
+     *                  <p> id,name,age </p>
+     *                  <p> 1,zhangsan,18   </p>
+     *                  <p> 2,lisi,20 </p>
+     *                  <p> ------------ </p>
+     * @param cls       需要转换的对象,含有<b>id,name,age</b>字段
+     * @param delimiter 分隔符
+     * @param <I>
+     * @return
+     */
+    public static <I> List<I> readCsv(String data, Class<I> cls, char delimiter) throws IOException {
+        CsvMapper csvMapper = new CsvMapper();
+        CsvSchema csvSchema = CsvSchema.emptySchema().withHeader().withColumnSeparator(delimiter);
+        MappingIterator<I> orderLines = csvMapper.readerFor(cls).with(csvSchema).readValues(data);
+        return orderLines.readAll();
+    }
+
+    /**
+     * 读取csv文件
+     *
+     * @param csvFilePath 文件地址
+     * @param cls         读取转化的对象
+     * @param hasHeader   是否存在header
+     * @param delimiter   分隔符.默认【,】
+     * @param <I>
+     * @return
+     */
+    public static <I> List<I> readCsv(final String csvFilePath, Class<I> cls, boolean hasHeader, char delimiter) {
+        if (!isFileExist(csvFilePath)) {
+            throw new RuntimeException(String.format("解析用的csv：\u0020[%s] 文件不存在。", csvFilePath));
+        }
+        if (StrUtil.isEmpty(delimiter)) {
+            delimiter = ',';
+        }
+        try {
+            File csvFile = new File(csvFilePath);
+            CsvMapper csvMapper = new CsvMapper();
+            CsvSchema csvSchema = csvMapper.typedSchemaFor(cls).withStrictHeaders(hasHeader).withColumnSeparator(delimiter).withComments();
+            MappingIterator<I> recordIterator = csvMapper.readerWithTypedSchemaFor(cls).with(csvSchema).readValues(csvFile);
+            return recordIterator.readAll();
+        } catch (IOException e) {
+            throw new RuntimeException("[" + csvFilePath + "] 文件解析失败。", e);
+        }
+    }
+
+    /**
+     * 判断文件是否存在
+     *
+     * @param filePath
+     * @return
+     */
+    public static boolean isFileExist(String filePath) {
+        if (StrUtil.isEmpty(filePath)) {
+            return false;
+        }
+
+        File file = new File(filePath);
+        return file.exists() && !file.isDirectory();
+    }
+
+    /**
+     * 分段读取大文件
+     *
+     * @param path         文件路径
+     * @param readFileBean 分段每次读取的bean 初始值需要设置每次读取的行数
+     * @param <T>          读取结果类型bean
+     * @return
+     */
+    public <T> ReadFileBean<T> readBigFile(String path, ReadFileBean<T> readFileBean, Class<T> readCls) {
+        File file = new File(path);
+        judeFileExists(file);
+        try {
+            RandomAccessFile randomAccessFile = new RandomAccessFile(file, "r");
+            randomAccessFile.seek(readFileBean.getPointer());
+            readFileBean.setFileLength(randomAccessFile.length());
+            List<String> datas = new ArrayList<>();
+            for (int i = 0; i < readFileBean.getReadRowNum(); i++) {
+                String s = randomAccessFile.readLine();
+                datas.add(charsetDecode(s, StandardCharsets.UTF_8));
+                readFileBean.setPointer(randomAccessFile.getFilePointer());
+            }
+            randomAccessFile.close();
+            if (path.contains(TYPE_CSV)) {
+                //csv文件处理
+                datas.add(0, getCsvHeader(readCls));
+                List<T> dataList = readCsv(String.join("\n", datas), readCls, ',');
+                readFileBean.setDatas(dataList);
+            }
+        } catch (FileNotFoundException e) {
+            logger.severe("file exists." + e.getMessage());
+        } catch (IOException e) {
+            logger.severe("read file error." + e.getMessage());
+        }
+        return readFileBean;
+    }
+
+    /**
+     * 获取csv的header,使用注解{@link CsvHeader}
+     *
+     * @param cls
+     * @param <T>
+     * @return
+     */
+    public static <T> String getCsvHeader(Class<T> cls) {
+        CsvHeader annotation = cls.getAnnotation(CsvHeader.class);
+        Assert.notNull(annotation, cls.toString() + "类上需要添加注解@CsvHeader，并指定header。");
+        String header = annotation.header();
+        Assert.notNull(header, cls.toString() + "类上需要添加注解@CsvHeader，并指定header。");
+        return header;
+    }
+
+    /**
+     * 把字符串data按照指定编码解码
+     *
+     * @param data    解码字符串
+     * @param charset 字符编码
+     * @return
+     */
+    public static String charsetDecode(String data, Charset charset) {
+        char[] chars = data.toCharArray();
+        byte[] result = new byte[chars.length];
+        for (int i = 0; i < chars.length; i++) {
+            result[i] = (byte) chars[i];
+        }
+        return new String(result, charset);
+    }
+
+
+    /**
+     * 提供读取IO流方法合集
+     * <p>读取速度快慢顺序：</p>
+     * 1. {@link #read4Buffer(InputStream, Charset)} <br>
+     * 2. {@link #read4IOUtilsCopy(InputStream, Charset)} <br>
+     * 3. {@link #read4ByteStreams(InputStream, Charset)} <br>
+     * 4. {@link #read4Byte(InputStream, Charset)} <br>
+     * 5. {@link #read4StringBuilder(InputStream, Charset)} <br>
+     * 6. {@link #read4BufferedReaderParallel(InputStream, Charset, String)}<br>
+     * 7. {@link #read4BufferedReader(InputStream, Charset, String)}<br>
+     * 8. {@link #read4ScannerA(InputStream)}<br>
+     * 9. {@link #read4BufferIO(InputStream, Charset)}<br>
+     * 10. {@link #read4IOUtils(InputStream, Charset)}<br>
+     * 11. {@link #read4ScannerZ(InputStream)}<br>
+     * 12. {@link #read4CharStreams(InputStream, Charset)}<br>
+     */
+    public static class FileInput {
+        /**
+         * 读取IO流内容：byte方式
+         *
+         * @param inputStream
+         * @param charset     编码：默认 {@link Charset#defaultCharset()}
+         * @return
+         * @throws IOException
+         */
+        public static String read4Byte(InputStream inputStream, Charset charset) throws IOException {
+            byte[] bytes = new byte[inputStream.available()];
+            inputStream.read(bytes);
+            return new String(bytes, charset == null ? Charset.defaultCharset() : charset);
+        }
+
+        /**
+         * 读取IO流内容：BufferedReader方式
+         *
+         * @param inputStream
+         * @param charset       编码：默认跟随系统 {@link Charset#defaultCharset()}
+         * @param lineSeparator 换行方式：默认跟随系统 {@link System#lineSeparator()}
+         * @return
+         */
+        public static String read4BufferedReader(InputStream inputStream, Charset charset, String lineSeparator) {
+            return new BufferedReader(
+                    new InputStreamReader(inputStream, charset == null ? Charset.defaultCharset() : charset)
+            ).lines().collect(Collectors.joining(lineSeparator == null ? System.lineSeparator() : lineSeparator));
+        }
+
+        /**
+         * 读取IO流内容：BufferedReader 并行方式
+         *
+         * @param inputStream
+         * @param charset       编码：默认跟随系统 {@link Charset#defaultCharset()}
+         * @param lineSeparator 换行方式：默认跟随系统 {@link System#lineSeparator()}
+         * @return
+         */
+        public static String read4BufferedReaderParallel(InputStream inputStream, Charset charset, String lineSeparator) {
+            return new BufferedReader(
+                    new InputStreamReader(inputStream, charset == null ? Charset.defaultCharset() : charset)
+            ).lines().parallel()
+                    .collect(Collectors.joining(lineSeparator == null ? System.lineSeparator() : lineSeparator));
+        }
+
+
+        /**
+         * 读取IO流内容：Scanner A方式
+         *
+         * @param inputStream
+         * @return
+         */
+        public static String read4ScannerA(InputStream inputStream) {
+            Scanner s = new Scanner(inputStream).useDelimiter("\\A");
+            String str = s.hasNext() ? s.next() : "";
+            return str;
+        }
+
+        /**
+         * 读取IO流内容：Scanner Z方式
+         *
+         * @param inputStream
+         * @return
+         */
+        public static String read4ScannerZ(InputStream inputStream) {
+            return new Scanner(inputStream).useDelimiter("\\Z").next();
+        }
+
+
+        /**
+         * 读取IO流内容：StringBuilder方式
+         *
+         * @param inputStream
+         * @param charset     编码：默认跟随系统 {@link Charset#defaultCharset()}
+         * @return
+         */
+        public static String read4StringBuilder(InputStream inputStream, Charset charset) throws IOException {
+            StringBuilder sb = new StringBuilder();
+            String line;
+            BufferedReader br = new BufferedReader(new InputStreamReader(inputStream, charset == null ? Charset.defaultCharset() : charset));
+            while ((line = br.readLine()) != null) {
+                sb.append(line);
+            }
+            return sb.toString();
+        }
+
+        /**
+         * 读取IO流内容：ByteArrayOutputStream方式
+         *
+         * @param inputStream
+         * @param charset     编码：默认跟随系统 {@link Charset#defaultCharset()}
+         * @return
+         */
+        public static String read4Buffer(InputStream inputStream, Charset charset) throws IOException {
+            ByteArrayOutputStream result = new ByteArrayOutputStream();
+            byte[] buffer = new byte[1024];
+            int length;
+            while ((length = inputStream.read(buffer)) != -1) {
+                result.write(buffer, 0, length);
+            }
+            return result.toString(charset == null ? Charset.defaultCharset().name() : charset.name());
+        }
+
+        /**
+         * 读取IO流内容：BufferedInputStream+ByteArrayOutputStream方式
+         *
+         * @param inputStream
+         * @param charset     编码：默认跟随系统 {@link Charset#defaultCharset()}
+         * @return
+         */
+        public static String read4BufferIO(InputStream inputStream, Charset charset) throws IOException {
+            BufferedInputStream bis = new BufferedInputStream(inputStream);
+            ByteArrayOutputStream buf = new ByteArrayOutputStream();
+            int result = bis.read();
+            while (result != -1) {
+                buf.write((byte) result);
+                result = bis.read();
+            }
+            return buf.toString(charset == null ? Charset.defaultCharset().name() : charset.name());
+        }
+
+        /**
+         * 读取IO流内容： 依赖于commons-io:commons-io {@link IOUtils#copy(Reader, OutputStream, Charset)}
+         *
+         * @param inputStream
+         * @param charset     编码：默认跟随系统 {@link Charset#defaultCharset()}
+         * @return
+         * @throws IOException
+         */
+        public static String read4IOUtilsCopy(InputStream inputStream, Charset charset) throws IOException {
+            StringWriter writer = new StringWriter();
+            IOUtils.copy(inputStream, writer, charset == null ? Charset.defaultCharset() : charset);
+            return writer.toString();
+        }
+
+        /**
+         * 读取IO流内容： 依赖于commons-io:commons-io {@link IOUtils#toString(InputStream, Charset)}
+         *
+         * @param inputStream
+         * @param charset     编码：默认跟随系统 {@link Charset#defaultCharset()}
+         * @return
+         * @throws IOException
+         */
+        public static String read4IOUtils(InputStream inputStream, Charset charset) throws IOException {
+            return IOUtils.toString(inputStream, charset == null ? Charset.defaultCharset() : charset);
+        }
+
+        /**
+         * 读取IO流内容： 依赖于com.google.guava:guava {@link CharStreams#toString(Readable)}
+         *
+         * @param inputStream
+         * @param charset     编码：默认跟随系统 {@link Charset#defaultCharset()}
+         * @return
+         * @throws IOException
+         */
+        public static String read4CharStreams(InputStream inputStream, Charset charset) throws IOException {
+            return CharStreams.toString(new InputStreamReader(inputStream, charset == null ? Charset.defaultCharset() : charset));
+        }
+
+        /**
+         * 读取IO流内容： 依赖于com.google.guava:guava {@link ByteStreams#toByteArray(InputStream)}
+         *
+         * @param inputStream
+         * @param charset     编码：默认跟随系统 {@link Charset#defaultCharset()}
+         * @return
+         * @throws IOException
+         */
+        public static String read4ByteStreams(InputStream inputStream, Charset charset) throws IOException {
+            return new String(ByteStreams.toByteArray(inputStream), charset == null ? Charset.defaultCharset() : charset);
+        }
+    }
+
     /*public static void main(String[] args) {
         long size = FileUtil.size(new File("E:\\OS\\deepin-15.6-amd64\\DeepinCloudPrintServerInstaller_1.0.0.1.exe"));
         System.out.println(size);

src/main/java/com/yexuejc/base/util/ObjUtil.java

@@ -1,5 +1,7 @@
 package com.yexuejc.base.util;
 
+import com.yexuejc.base.annotation.ToUeProperty;
+
 import java.io.*;
 import java.lang.reflect.Field;
 import java.time.LocalDate;

src/test/java/com/yexuejc/base/util/ObjUtilTest.java

@@ -1,5 +1,7 @@
 package com.yexuejc.base.util;
 
+import com.yexuejc.base.annotation.ToUeProperty;
+
 import java.io.Serializable;
 import java.math.BigDecimal;
 import java.time.LocalDateTime;