JAVA实现大文件排序

1、ackage com.scott.util;import java.io.*;import java.util.ArrayList;import java.util.Comparator;import java.util.Iterator;import java.util.List;/* Created by Scott on 2017/11/1.*/public class LargeFileDataSort /测试大文件路径public final static String testFilePath = E:/dataTest/largeFileData.txt;public final

2、 static String resultFilePath = E:/dataTest/largeFileResult.txt;/切分大文件的小文件大小MB,默认为100MBprivate final static int size = 200;private static int byteSize = size * 1024 * 1024;public static void main(String args) throws IOException /生成测试文件createTestData();Long start = System.currentTimeMillis();work();L

3、ong end = System.currentTimeMillis();System.out.println(end - start) / 1000/ 60);/*切分文件每份大小*/public static void work() throws IOException File file = new File(testFilePath);if (!file.exists() return;)/ 2.1得到文件大小MBdouble mbsize = file.length() / 1024 / 1024;/ 2.2计算得到切分的文件数double fileNum = Math.ceil(m

4、bsize / size);/ 2.3临时文件List tempFileList = createTempFileList(file, fileNum);/ 2.3切分文件divAndFirstSort(file, tempFileList);BufferedWriter的 flush()方法).append();BufferedWriter的 flush()方法).append();BufferedWriter的 flush()方法).append();BufferedWriter的 flush()方法).append();BufferedWriter的 flush()方法).append(

5、);BufferedWriter的 flush()方法).append();BufferedWriter的 flush()方法).append();BufferedWriter的 flush()方法).append();BufferedWriter的 flush()方法).append();BufferedWriter的 flush()方法).append();BufferedWriter的 flush()方法).append();BufferedWriter的 flush()方法).append();BufferedWriter的 flush()方法).append();BufferedWr

6、iter的 flush()方法).append();BufferedWriter的 flush()方法).append();BufferedWriter的 flush()方法).append();/ 2.4递归排序(每个文件读取多少数据放到内存排序后合并到结果文件)/排序合并开始mergeLargeFile(tempFileList);/ 2.5 TODO把临时文件删除)/*生成测试文件*/public static void createTestData() StringBuffer sb = new StringBuffer();BufferedWriter bw = null;try F

7、ile testFile = new File(testFilePath);if (!testFile.exists() testFile.createNewFile();)OutputStreamWriter(newbw = new BufferedWriter(new FileOutputStream(testFile);for (long i = 1; i = 27777777; i+) sb.setLength(0);sb.append(i).append();/sb.append(random.nextInt(100000).append();sb.append(i).append(

8、调用 sb.append(i).append(调用 sb.append(i).append(调用 sb.append(i).append(调用 sb.append(i).append(调用 sb.append(i).append(调用 sb.append(i).append(调用 sb.append(i).append(调用 sb.append(i).append(调用 sb.append(i).append(调用 sb.append(i).append(调用 sb.append(i).append(调用 sb.append(i).append(调用 sb.append(i).append(调

9、用 sb.append(i).append(调用 sb.append(i).append(调用 sb.append(n);bw.write(sb.toString();if (i + 1) % 5000 = 0) bw.flush();System.out.println(i);) catch (lOException e) System.out.println(生成测试文件失败！ + e.getMessage(); finally try if (bw != null) bw.close(); catch (lOException e) /*把临时文件合并到结果文件param tempFil

10、eListthrows lOException/public static void mergeLargeFile(List tempFileList) throws lOException List bwList = new ArrayList();for(int i=0; i tempFileList.size(); i+) FileEntity le = new FileEntity(new BufferedReader(new InputStreamReader(new FilelnputStream(tempFileList.get(i);bwList.add(le);Buffere

11、dWriter resultBw = null;try resultBw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(resultFilePath);Long count = 0L;FileEntity fe = null;while (fe = getFirstFileEntity(bwList) != null) System.out.println(- 写文件 id: + fe.getId();/写入符合条件的一行数据resultBw.write(fe.getLine() + n);/准备下一行fe.n

12、extLine();/清缓冲流if (count % 1000 = 0) resultBw.flush(); catch (Exception e) finally if (resultBw != null) try resultBw.close(); catch (IOException e) /关闭for(int i=0; i bwList.size(); i+) bwList.get(i).close();/*从切分的文件中按序行读取(因为切分文件时已经做好了排序)param bwListreturn/private static FileEntity getFirstFileEntit

13、y(List bwList) if (bwList.size() = 0) return null;Iterator it = bwList.iterator();while (it.hasNext() FileEntity fe = it.next();/如果文件读到完就关闭流和删除在集合的文件流if (fe.getLine() = null) fe.close(); it.remove();if (bwList.size() = 0) return null;/排序获取一行数据bwList.sort(new FileEntityComparator();/返回第一个符合条件的文件对象ret

14、urn bwList.get(0);/*切分文件并做第一次排序* param file* param tempFileList*/private static void divAndFirstSort(File file, List tempFileList) BufferedReader br = null;try /读取大文件br = new BufferedReader(new InputStreamReader(new FileInputStream(file); /行数据保存对象String line = null;/临时文件索引int index = tempFileList.si

15、ze() - 1;/第一个临时文件File tempFile = tempFileList.get(index);List lineList = new ArrayList();int byteSum = 0;/循环临时文件并循环大文件while (line = br.readLine() != null) line += n;byteSum += line.getBytes().length;/如果长度达到每个文件大小则重新计算if (byteSum = byteSize) /写入到文件putLineListToFile(tempFileList.get(index), lineList);

16、index-;byteSum = line.getBytes().length;lineList.clear();lineList.add(line);if (lineList.size() 0) /写入到文件putLineListToFile(tempFileList.get(0), lineList); catch (FileNotFoundException e) e.printStackTrace(); catch (IOException e) e.printStackTrace(); finally try if (br != null) br.close(); catch (IO

17、Exception e) )/*把数据写到临时文件param lineList*/private static void putLineListToFile(File file, List lineList) throws lOException FileOutputStream tempFileFos = null;try /很关键的一步，第一次写入文件必须排序lineList.sort(new LineComparator();tempFileFos = new FileOutputStream(file);for(int i=0; i lineList.size(); i+) tempF

18、ileFos.write(lineList.get(i).getBytes();) finally if (tempFileFos != null) tempFileFos.close();/*生成临时文件param fileNumreturn/private static List createTempFileList(File file, double fileNum) List tempFileList = new ArrayList();String fileFolder = file.getParent();String name = file.getName();for (int

19、i = 0; i fileNum; i+) File tempFile = new File(fileFolder + / + name + .temp_ + i + .txt);if (tempFile.exists() tempFile.delete(); try tempFile.createNewFile(); catch (IOException e) e.printStackTrace();tempFileList.add(tempFile);return tempFileList;public static int compare(String o1, String o2) St

20、ring 011d = o1.substring(0, o1.indexOf();String o2Id = o2.substring(0, o2.indexOf();/从小到大return Integer.parseInt(o1Id) - Integer.parseInt(o2Id);/从大到小/return Integer.parseInt(o2Id) - Integer.parseInt(o1Id);/*排序*/class LineComparator implements Comparator Overridepublic int compare(String o1, String o2) return LargeFileDataSpare(o1, o2);)/*排序类*/class FileEntityComparator implements Comparator Overridepublic int compare(FileEntity o1, FileEntity o2) return LargeFileDataSpare(o1.getLine(), o2.getLine(