From 9731faf9f8d3a6c4995c55d5b17bef92e3b7cd0d Mon Sep 17 00:00:00 2001 From: its Date: Mon, 15 Apr 2024 18:16:00 +0800 Subject: [PATCH] =?UTF-8?q?[update]=20FileUtil=E5=A2=9E=E5=8A=A0=E8=AF=BB?= =?UTF-8?q?=E5=8F=96=E5=A4=A7=E6=96=87=E4=BB=B6=E8=87=AA=E5=AE=9A=E4=B9=89?= =?UTF-8?q?=E6=96=B9=E6=B3=95=E5=92=8C=E5=8D=95=E7=BA=AF=E8=AF=BB=E5=8F=96?= =?UTF-8?q?=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- UPDATE.md | 8 ++ .../IntegerNullValueDeserializer.java | 38 ++++++++ .../java/com/yexuejc/base/util/FileUtil.java | 93 ++++++++++++------- .../com/yexuejc/base/util/FileUtilTest.java | 57 ++++++++++-- .../base/util/bean/AppnodeCertCsvBean.java | 4 + 5 files changed, 163 insertions(+), 37 deletions(-) create mode 100644 src/main/java/com/yexuejc/base/converter/IntegerNullValueDeserializer.java diff --git a/UPDATE.md b/UPDATE.md index 8263feb..2190d61 100644 --- a/UPDATE.md +++ b/UPDATE.md @@ -1,5 +1,13 @@ yexuejc-base 更新记录 ------------------ +#### version ：1.5.3-jre8 +**time： **
+**branch：** jre8
+**update：**
+1. [FileUtil.java](src/main/java/com/yexuejc/base/util/FileUtil.java)增加读取大文件自定义方法和单纯读取方法 +2. +--- + #### version ：1.5.2-jre8 **time：2024-4-7 14:34:33**
**branch：** jre8
diff --git a/src/main/java/com/yexuejc/base/converter/IntegerNullValueDeserializer.java b/src/main/java/com/yexuejc/base/converter/IntegerNullValueDeserializer.java new file mode 100644 index 0000000..89e0e05 --- /dev/null +++ b/src/main/java/com/yexuejc/base/converter/IntegerNullValueDeserializer.java @@ -0,0 +1,38 @@ +package com.yexuejc.base.converter; + +import java.io.IOException; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.StdScalarDeserializer; + +/** + * 反序列化中内容为空时，返回Integer为空 + *

使用方式：@JsonDeserialize(using = IntegerNullValueDeserializer.class)

+ * @author: yexuejc + * @date: 2024/4/15 18:08 + */ +public class IntegerNullValueDeserializer extends StdScalarDeserializer { + public IntegerNullValueDeserializer() { + super(Integer.class); + } + + @Override + public Integer deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { + String value = p.getValueAsString(); + if (isInteger(value)) { + return super._parseInteger(p, ctxt, Integer.class); + } else { + return null; + } + } + + private static boolean isInteger(String s) { + try { + Integer.parseInt(s); + return true; + } catch (NumberFormatException e) { + return false; + } + } +} \ No newline at end of file diff --git a/src/main/java/com/yexuejc/base/util/FileUtil.java b/src/main/java/com/yexuejc/base/util/FileUtil.java index b55f6c4..5207643 100644 --- a/src/main/java/com/yexuejc/base/util/FileUtil.java +++ b/src/main/java/com/yexuejc/base/util/FileUtil.java @@ -18,6 +18,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; import java.util.List; +import java.util.function.Function; import java.util.logging.Level; import java.util.logging.Logger; import java.util.zip.CRC32; @@ -38,11 +39,14 @@ import io.jsonwebtoken.lang.Assert; * @time 2017年11月3日 下午3:12:49 */ public class FileUtil { - static Logger logger = Logger.getLogger(FileUtil.class.getName()); + private static final Logger logger = Logger.getLogger(FileUtil.class.getName()); private FileUtil() { } + + private static final String NEW_LINE = "\n"; + private static final String CONSTANT_DOT = "."; private static final String TYPE_TAR_GZ = ".tar.gz"; private static final String TYPE_CSV = ".csv"; private static final String TAR_GZ = "tar.gz"; @@ -61,7 +65,7 @@ public class FileUtil { if (fileName.lastIndexOf(TYPE_TAR_GZ) > 0) { return TAR_GZ; } - return fileName.substring(fileName.lastIndexOf(".") + 1); + return fileName.substring(fileName.lastIndexOf(CONSTANT_DOT) + 1); } catch (Exception e) { logger.severe("file doesn't exist or is not a file"); } @@ -332,22 +336,19 @@ public class FileUtil { } /** - * 分段读取大文件 + * 分段读取大文件（不限格式） * - * @param csvFilePath 文件路径 + * @param filePath 文件路径 * @param readFileBean 分段每次读取的bean 初始值需要设置每次读取的行数 * @param 读取结果类型bean - * @return + * @return 文件分页读取内容（自定义处理后）及读取信息 */ - public static ReadFileBean readBigFile(String csvFilePath, ReadFileBean readFileBean, Class readCls) throws IOException { - if (!isFileExist(csvFilePath)) { - throw new FileNotFoundException(String.format("解析用的csv： [%s] 文件不存在。", csvFilePath)); - } - if (!csvFilePath.endsWith(TYPE_CSV)) { - throw new IOException(String.format("解析用的csv： [%s] 文件不是CSV文件格式。", csvFilePath)); + public static ReadFileBean readBigFile(String filePath, ReadFileBean readFileBean, Function, List> readAfter) throws IOException { + if (!isFileExist(filePath)) { + throw new FileNotFoundException(String.format("[%s]文件不存在。", filePath)); } List datas = new ArrayList<>(); - try (RandomAccessFile randomAccessFile = new RandomAccessFile(new File(csvFilePath), "r")) { + try (RandomAccessFile randomAccessFile = new RandomAccessFile(new File(filePath), "r")) { if (readFileBean.getPointer() < 0) { readFileBean.setPointer(0); } @@ -365,29 +366,59 @@ public class FileUtil { //无数据 return readFileBean.setDatas(new ArrayList<>()); } - - //csv文件处理 - com.yexuejc.base.pojo.CsvToBean csvToBean = getCsvToBean(readCls); - readFileBean.setHeader(csvToBean.getHeader()); - if (csvToBean.hasHeader()) { - //文件存在header,设置header优先,没设置使用文件的 - if (StrUtil.isNotEmpty(csvToBean.getHeader())) { - //替换header - datas.remove(0); - datas.add(0, csvToBean.getHeader()); - } else { - readFileBean.setHeader(datas.get(0)); - } - } else { - //文件不存在header，使用设置的 - datas.add(0, csvToBean.getHeader()); - } - - List dataList = readCsv(String.join("\n", datas), readCls, csvToBean.getDelimiter()); + List dataList = readAfter.apply(datas); readFileBean.setDatas(dataList); return readFileBean; } + /** + * 分段读取大文件(不解析) + * + * @param csvFilePath 文件路径 + * @param readFileBean 分段每次读取的bean 初始值需要设置每次读取的行数 + * @return 文件分页读取内容（每行为一个String对象）及读取信息 + */ + public static ReadFileBean readBigFile(String csvFilePath, ReadFileBean readFileBean) throws IOException { + return readBigFile(csvFilePath, readFileBean, (datas) -> datas); + } + + /** + * 分段读取大文件(CSV格式) + * + * @param csvFilePath 文件路径 + * @param readFileBean 分段每次读取的bean 初始值需要设置每次读取的行数 + * @param 读取结果类型bean + * @return 文件分页读取内容（转bean后）及读取信息 + */ + public static ReadFileBean readBigFile(String csvFilePath, ReadFileBean readFileBean, Class readCls) throws IOException { + if (!csvFilePath.endsWith(TYPE_CSV)) { + throw new IOException(String.format("[%s]文件不是CSV文件格式。", csvFilePath)); + } + return readBigFile(csvFilePath, readFileBean, (datas) -> { + //csv文件处理 + com.yexuejc.base.pojo.CsvToBean csvToBean = getCsvToBean(readCls); + readFileBean.setHeader(csvToBean.getHeader()); + if (csvToBean.hasHeader()) { + //文件存在header,设置header优先,没设置使用文件的 + if (StrUtil.isNotEmpty(csvToBean.getHeader())) { + //替换header + datas.remove(0); + datas.add(0, csvToBean.getHeader()); + } else { + readFileBean.setHeader(datas.get(0)); + } + } else { + //文件不存在header，使用设置的 + datas.add(0, csvToBean.getHeader()); + } + try { + return readCsv(String.join(NEW_LINE, datas), readCls, csvToBean.getDelimiter()); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + /** * 获取csv的header,使用注解{@link CsvToBean} * diff --git a/src/test/java/com/yexuejc/base/util/FileUtilTest.java b/src/test/java/com/yexuejc/base/util/FileUtilTest.java index e376e46..da8e212 100644 --- a/src/test/java/com/yexuejc/base/util/FileUtilTest.java +++ b/src/test/java/com/yexuejc/base/util/FileUtilTest.java @@ -4,6 +4,7 @@ import java.io.File; import java.io.IOException; import java.nio.file.Paths; import java.util.List; +import java.util.stream.Collectors; import com.yexuejc.base.pojo.ReadFileBean; import com.yexuejc.base.util.bean.AppnodeCertCsvBean; @@ -54,19 +55,63 @@ public class FileUtilTest { } private static void readCsvFile() throws IOException { - String path = "F:\\coding\\yexuejc-base2\\src\\test\\java\\com\\yexuejc\\base\\util\\test.csv"; + String path = "F:\\coding\\yexuejc-base\\src\\test\\java\\com\\yexuejc\\base\\util\\test.csv"; - List list = FileUtil.readCsv(path, AppnodeCertCsvBean.class, true, "enable,domain,protocol,deployHost,deployPath,uname,pwd,appnodeId", ','); - System.out.println("***********************************************"); - System.out.println(JsonUtil.formatPrinter(list)); - System.out.println("条数：" + list.size()); +// List list = FileUtil.readCsv(path, AppnodeCertCsvBean.class, true, "enable,domain,protocol,deployHost,deployPath,uname,pwd,appnodeId", ','); +// System.out.println("***********************************************"); +// System.out.println(JsonUtil.formatPrinter(list)); +// System.out.println("条数：" + list.size()); + //直接把每行读取成字符串 + ReadFileBean readFileBean2 = new ReadFileBean<>(2); + ReadFileBean bean2 = FileUtil.readBigFile(path, readFileBean2); + System.out.println("直接把每行读取成字符串============================================"); + System.out.println(JsonUtil.formatPrinter(bean2)); + System.out.println("直接把每行读取成字符串============================================"); + + //自定义每行数据的处理 + ReadFileBean readFileBean1 = new ReadFileBean<>(2); + ReadFileBean bean1 = FileUtil.readBigFile(path, readFileBean1, datas -> { + if (readFileBean1.getStartRowNum() == 1) { + datas.remove(0);//跳过第一行 + } + return datas.stream().map(str -> { + //自定义处理每一条数据 + String[] split = str.split(","); + AppnodeCertCsvBean app = new AppnodeCertCsvBean(); + app.setEnable(getValue(split, 0)); + app.setDomain(getValue(split, 1)); + app.setProtocol(getValue(split, 2)); + app.setDeployHost(getValue(split, 3)); + app.setDeployPath(getValue(split, 4)); + app.setUname(getValue(split, 5)); + app.setPwd(getValue(split, 6)); + if (StrUtil.isNotEmpty(getValue(split, 7))) { + app.setAppnodeId(Integer.valueOf(getValue(split, 7))); + } + return app; + }).collect(Collectors.toList()); + }); + System.out.println("自定义每行数据的处理============================================"); + System.out.println(JsonUtil.formatPrinter(bean1)); + System.out.println("自定义每行数据的处理============================================"); + + //直接使用提供的csv文件读取 ReadFileBean readFileBean = new ReadFileBean<>(2); do { ReadFileBean bean = FileUtil.readBigFile(path, readFileBean, AppnodeCertCsvBean.class); - System.out.println("============================================"); + System.out.println("直接使用提供的csv文件读取============================================"); System.out.println(JsonUtil.formatPrinter(bean)); + System.out.println("直接使用提供的csv文件读取============================================"); } while (readFileBean.hasNext()); } + + private static String getValue(String[] value, int index) { + try { + return value[index]; + } catch (Exception e) { + return ""; + } + } } diff --git a/src/test/java/com/yexuejc/base/util/bean/AppnodeCertCsvBean.java b/src/test/java/com/yexuejc/base/util/bean/AppnodeCertCsvBean.java index 32ad4e5..3c5e51c 100644 --- a/src/test/java/com/yexuejc/base/util/bean/AppnodeCertCsvBean.java +++ b/src/test/java/com/yexuejc/base/util/bean/AppnodeCertCsvBean.java @@ -2,13 +2,16 @@ package com.yexuejc.base.util.bean; import java.io.Serializable; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.yexuejc.base.annotation.CsvToBean; +import com.yexuejc.base.converter.IntegerNullValueDeserializer; /** * * @author: yexuejc * @date: 2024/2/27 10:40 */ +@CsvToBean(header = "enable,domain,protocol,deployHost,deployPath,uname,pwd,appnodeId") public class AppnodeCertCsvBean implements Serializable { /**是否生效：Y/N*/ private String enable; @@ -36,6 +39,7 @@ public class AppnodeCertCsvBean implements Serializable { /** * appnode协议时：且远程部署时，对应的远程appnode的ApiNodeId */ + @JsonDeserialize(using = IntegerNullValueDeserializer.class) private Integer appnodeId; public String getEnable() {