diff --git a/UPDATE.md b/UPDATE.md
index 8263feb..2190d61 100644
--- a/UPDATE.md
+++ b/UPDATE.md
@@ -1,5 +1,13 @@
yexuejc-base 更新记录
------------------
+#### version ：1.5.3-jre8
+**time： **
+**branch：** jre8
+**update：**
+1. [FileUtil.java](src/main/java/com/yexuejc/base/util/FileUtil.java)增加读取大文件自定义方法和单纯读取方法
+2.
+---
+
#### version ：1.5.2-jre8
**time：2024-4-7 14:34:33**
**branch：** jre8
diff --git a/src/main/java/com/yexuejc/base/converter/IntegerNullValueDeserializer.java b/src/main/java/com/yexuejc/base/converter/IntegerNullValueDeserializer.java
new file mode 100644
index 0000000..89e0e05
--- /dev/null
+++ b/src/main/java/com/yexuejc/base/converter/IntegerNullValueDeserializer.java
@@ -0,0 +1,38 @@
+package com.yexuejc.base.converter;
+
+import java.io.IOException;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.deser.std.StdScalarDeserializer;
+
+/**
+ * 反序列化中内容为空时，返回Integer为空
+ * 使用方式：@JsonDeserialize(using = IntegerNullValueDeserializer.class)
+ * @author: yexuejc
+ * @date: 2024/4/15 18:08
+ */
+public class IntegerNullValueDeserializer extends StdScalarDeserializer {
+ public IntegerNullValueDeserializer() {
+ super(Integer.class);
+ }
+
+ @Override
+ public Integer deserialize(JsonParser p, DeserializationContext ctxt) throws IOException {
+ String value = p.getValueAsString();
+ if (isInteger(value)) {
+ return super._parseInteger(p, ctxt, Integer.class);
+ } else {
+ return null;
+ }
+ }
+
+ private static boolean isInteger(String s) {
+ try {
+ Integer.parseInt(s);
+ return true;
+ } catch (NumberFormatException e) {
+ return false;
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/com/yexuejc/base/util/FileUtil.java b/src/main/java/com/yexuejc/base/util/FileUtil.java
index b55f6c4..5207643 100644
--- a/src/main/java/com/yexuejc/base/util/FileUtil.java
+++ b/src/main/java/com/yexuejc/base/util/FileUtil.java
@@ -18,6 +18,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.List;
+import java.util.function.Function;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.zip.CRC32;
@@ -38,11 +39,14 @@ import io.jsonwebtoken.lang.Assert;
* @time 2017年11月3日 下午3:12:49
*/
public class FileUtil {
- static Logger logger = Logger.getLogger(FileUtil.class.getName());
+ private static final Logger logger = Logger.getLogger(FileUtil.class.getName());
private FileUtil() {
}
+
+ private static final String NEW_LINE = "\n";
+ private static final String CONSTANT_DOT = ".";
private static final String TYPE_TAR_GZ = ".tar.gz";
private static final String TYPE_CSV = ".csv";
private static final String TAR_GZ = "tar.gz";
@@ -61,7 +65,7 @@ public class FileUtil {
if (fileName.lastIndexOf(TYPE_TAR_GZ) > 0) {
return TAR_GZ;
}
- return fileName.substring(fileName.lastIndexOf(".") + 1);
+ return fileName.substring(fileName.lastIndexOf(CONSTANT_DOT) + 1);
} catch (Exception e) {
logger.severe("file doesn't exist or is not a file");
}
@@ -332,22 +336,19 @@ public class FileUtil {
}
/**
- * 分段读取大文件
+ * 分段读取大文件（不限格式）
*
- * @param csvFilePath 文件路径
+ * @param filePath 文件路径
* @param readFileBean 分段每次读取的bean 初始值需要设置每次读取的行数
* @param 读取结果类型bean
- * @return
+ * @return 文件分页读取内容（自定义处理后）及读取信息
*/
- public static ReadFileBean readBigFile(String csvFilePath, ReadFileBean readFileBean, Class readCls) throws IOException {
- if (!isFileExist(csvFilePath)) {
- throw new FileNotFoundException(String.format("解析用的csv： [%s] 文件不存在。", csvFilePath));
- }
- if (!csvFilePath.endsWith(TYPE_CSV)) {
- throw new IOException(String.format("解析用的csv： [%s] 文件不是CSV文件格式。", csvFilePath));
+ public static ReadFileBean readBigFile(String filePath, ReadFileBean readFileBean, Function, List> readAfter) throws IOException {
+ if (!isFileExist(filePath)) {
+ throw new FileNotFoundException(String.format("[%s]文件不存在。", filePath));
}
List datas = new ArrayList<>();
- try (RandomAccessFile randomAccessFile = new RandomAccessFile(new File(csvFilePath), "r")) {
+ try (RandomAccessFile randomAccessFile = new RandomAccessFile(new File(filePath), "r")) {
if (readFileBean.getPointer() < 0) {
readFileBean.setPointer(0);
}
@@ -365,29 +366,59 @@ public class FileUtil {
//无数据
return readFileBean.setDatas(new ArrayList<>());
}
-
- //csv文件处理
- com.yexuejc.base.pojo.CsvToBean csvToBean = getCsvToBean(readCls);
- readFileBean.setHeader(csvToBean.getHeader());
- if (csvToBean.hasHeader()) {
- //文件存在header,设置header优先,没设置使用文件的
- if (StrUtil.isNotEmpty(csvToBean.getHeader())) {
- //替换header
- datas.remove(0);
- datas.add(0, csvToBean.getHeader());
- } else {
- readFileBean.setHeader(datas.get(0));
- }
- } else {
- //文件不存在header，使用设置的
- datas.add(0, csvToBean.getHeader());
- }
-
- List dataList = readCsv(String.join("\n", datas), readCls, csvToBean.getDelimiter());
+ List dataList = readAfter.apply(datas);
readFileBean.setDatas(dataList);
return readFileBean;
}
+ /**
+ * 分段读取大文件(不解析)
+ *
+ * @param csvFilePath 文件路径
+ * @param readFileBean 分段每次读取的bean 初始值需要设置每次读取的行数
+ * @return 文件分页读取内容（每行为一个String对象）及读取信息
+ */
+ public static ReadFileBean readBigFile(String csvFilePath, ReadFileBean readFileBean) throws IOException {
+ return readBigFile(csvFilePath, readFileBean, (datas) -> datas);
+ }
+
+ /**
+ * 分段读取大文件(CSV格式)
+ *
+ * @param csvFilePath 文件路径
+ * @param readFileBean 分段每次读取的bean 初始值需要设置每次读取的行数
+ * @param 读取结果类型bean
+ * @return 文件分页读取内容（转bean后）及读取信息
+ */
+ public static ReadFileBean readBigFile(String csvFilePath, ReadFileBean readFileBean, Class readCls) throws IOException {
+ if (!csvFilePath.endsWith(TYPE_CSV)) {
+ throw new IOException(String.format("[%s]文件不是CSV文件格式。", csvFilePath));
+ }
+ return readBigFile(csvFilePath, readFileBean, (datas) -> {
+ //csv文件处理
+ com.yexuejc.base.pojo.CsvToBean csvToBean = getCsvToBean(readCls);
+ readFileBean.setHeader(csvToBean.getHeader());
+ if (csvToBean.hasHeader()) {
+ //文件存在header,设置header优先,没设置使用文件的
+ if (StrUtil.isNotEmpty(csvToBean.getHeader())) {
+ //替换header
+ datas.remove(0);
+ datas.add(0, csvToBean.getHeader());
+ } else {
+ readFileBean.setHeader(datas.get(0));
+ }
+ } else {
+ //文件不存在header，使用设置的
+ datas.add(0, csvToBean.getHeader());
+ }
+ try {
+ return readCsv(String.join(NEW_LINE, datas), readCls, csvToBean.getDelimiter());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
/**
* 获取csv的header,使用注解{@link CsvToBean}
*
diff --git a/src/test/java/com/yexuejc/base/util/FileUtilTest.java b/src/test/java/com/yexuejc/base/util/FileUtilTest.java
index e376e46..da8e212 100644
--- a/src/test/java/com/yexuejc/base/util/FileUtilTest.java
+++ b/src/test/java/com/yexuejc/base/util/FileUtilTest.java
@@ -4,6 +4,7 @@ import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.List;
+import java.util.stream.Collectors;
import com.yexuejc.base.pojo.ReadFileBean;
import com.yexuejc.base.util.bean.AppnodeCertCsvBean;
@@ -54,19 +55,63 @@ public class FileUtilTest {
}
private static void readCsvFile() throws IOException {
- String path = "F:\\coding\\yexuejc-base2\\src\\test\\java\\com\\yexuejc\\base\\util\\test.csv";
+ String path = "F:\\coding\\yexuejc-base\\src\\test\\java\\com\\yexuejc\\base\\util\\test.csv";
- List list = FileUtil.readCsv(path, AppnodeCertCsvBean.class, true, "enable,domain,protocol,deployHost,deployPath,uname,pwd,appnodeId", ',');
- System.out.println("***********************************************");
- System.out.println(JsonUtil.formatPrinter(list));
- System.out.println("条数：" + list.size());
+// List list = FileUtil.readCsv(path, AppnodeCertCsvBean.class, true, "enable,domain,protocol,deployHost,deployPath,uname,pwd,appnodeId", ',');
+// System.out.println("***********************************************");
+// System.out.println(JsonUtil.formatPrinter(list));
+// System.out.println("条数：" + list.size());
+ //直接把每行读取成字符串
+ ReadFileBean readFileBean2 = new ReadFileBean<>(2);
+ ReadFileBean bean2 = FileUtil.readBigFile(path, readFileBean2);
+ System.out.println("直接把每行读取成字符串============================================");
+ System.out.println(JsonUtil.formatPrinter(bean2));
+ System.out.println("直接把每行读取成字符串============================================");
+
+ //自定义每行数据的处理
+ ReadFileBean readFileBean1 = new ReadFileBean<>(2);
+ ReadFileBean bean1 = FileUtil.readBigFile(path, readFileBean1, datas -> {
+ if (readFileBean1.getStartRowNum() == 1) {
+ datas.remove(0);//跳过第一行
+ }
+ return datas.stream().map(str -> {
+ //自定义处理每一条数据
+ String[] split = str.split(",");
+ AppnodeCertCsvBean app = new AppnodeCertCsvBean();
+ app.setEnable(getValue(split, 0));
+ app.setDomain(getValue(split, 1));
+ app.setProtocol(getValue(split, 2));
+ app.setDeployHost(getValue(split, 3));
+ app.setDeployPath(getValue(split, 4));
+ app.setUname(getValue(split, 5));
+ app.setPwd(getValue(split, 6));
+ if (StrUtil.isNotEmpty(getValue(split, 7))) {
+ app.setAppnodeId(Integer.valueOf(getValue(split, 7)));
+ }
+ return app;
+ }).collect(Collectors.toList());
+ });
+ System.out.println("自定义每行数据的处理============================================");
+ System.out.println(JsonUtil.formatPrinter(bean1));
+ System.out.println("自定义每行数据的处理============================================");
+
+ //直接使用提供的csv文件读取
ReadFileBean readFileBean = new ReadFileBean<>(2);
do {
ReadFileBean bean = FileUtil.readBigFile(path, readFileBean, AppnodeCertCsvBean.class);
- System.out.println("============================================");
+ System.out.println("直接使用提供的csv文件读取============================================");
System.out.println(JsonUtil.formatPrinter(bean));
+ System.out.println("直接使用提供的csv文件读取============================================");
} while (readFileBean.hasNext());
}
+
+ private static String getValue(String[] value, int index) {
+ try {
+ return value[index];
+ } catch (Exception e) {
+ return "";
+ }
+ }
}
diff --git a/src/test/java/com/yexuejc/base/util/bean/AppnodeCertCsvBean.java b/src/test/java/com/yexuejc/base/util/bean/AppnodeCertCsvBean.java
index 32ad4e5..3c5e51c 100644
--- a/src/test/java/com/yexuejc/base/util/bean/AppnodeCertCsvBean.java
+++ b/src/test/java/com/yexuejc/base/util/bean/AppnodeCertCsvBean.java
@@ -2,13 +2,16 @@ package com.yexuejc.base.util.bean;
import java.io.Serializable;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.yexuejc.base.annotation.CsvToBean;
+import com.yexuejc.base.converter.IntegerNullValueDeserializer;
/**
*
* @author: yexuejc
* @date: 2024/2/27 10:40
*/
+@CsvToBean(header = "enable,domain,protocol,deployHost,deployPath,uname,pwd,appnodeId")
public class AppnodeCertCsvBean implements Serializable {
/**是否生效：Y/N*/
private String enable;
@@ -36,6 +39,7 @@ public class AppnodeCertCsvBean implements Serializable {
/**
* appnode协议时：且远程部署时，对应的远程appnode的ApiNodeId
*/
+ @JsonDeserialize(using = IntegerNullValueDeserializer.class)
private Integer appnodeId;
public String getEnable() {