nio按行读文件

向右看齐 2022-06-15 01:49 168阅读 0赞

不废话，直接上代码

package com.syz.test.nio;
    
    import java.io.IOException;
    import java.io.RandomAccessFile;
    import java.nio.ByteBuffer;
    import java.nio.channels.FileChannel;
    
    
    public class DataReader { 
        /** * 按行读，行的分隔符必须是单个字符"\n" */
        public static void readByLine(String filePath,String encoding,int bufferSize) {
            byte lf = 10;//"\n"
            RandomAccessFile raf = null;
            FileChannel fc = null;
            try {
                raf = new RandomAccessFile(filePath, "r");
                fc = raf.getChannel();
                ByteBuffer buffer = ByteBuffer.allocate(bufferSize);
                //保存换行后未处理的数据
                byte[] temp = new byte[0];
                while (fc.read(buffer)!= -1) {
                    int position = buffer.position();// 读取结束后的位置，相当于读取的长度
                    byte[] bs = new byte[position];// 用来存放读取的内容的数组
                    buffer.rewind();// 将position设回0,所以你可以重读Buffer中的所有数据,此处如果不设置,无法使用下面的get方法
                    buffer.get(bs);// 相当于buffer.get(bs,0,bs.length())：从position初始位置开始相对读,读bs.length个byte,并写入bs[0]到bs[bs.length-1]的区域
                    buffer.clear();
                    //要加上上次未处理的数据
                    int readSize = position + temp.length;
                    byte[] readByte = new byte[readSize];
                    System.arraycopy(temp, 0, readByte, 0, temp.length);
                    System.arraycopy(bs, 0, readByte, temp.length, position);
                    int start = 0;
                    int lineByteSize = 0;
                    byte[] lineByte = new byte[0];
                    for (int i = 0; i < readSize; i++) {
                        if (readByte[i] == lf) {
                            lineByte = new byte[lineByteSize];
                            System.arraycopy(readByte, start, lineByte, 0, lineByteSize);
                            String lineStr = new String(lineByte, encoding);
                            System.out.println(lineStr);
                            lineByteSize=0;
                            start = i+1;
                        } else {
                            lineByteSize++;
                        }
    
                    }
                    // 把剩余的放入临时数组中
                    temp = new byte[lineByteSize];
                    System.arraycopy(readByte, start, temp, 0, lineByteSize);
                }
                //处理临时数组中的数据
                if(temp.length>0){
                    String lineStr = new String(temp, encoding);
                    System.out.println(lineStr);
                }
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                if (fc != null) {
                    try {
                        fc.close();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                }
                if (raf != null) {
                    try {
                        raf.close();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                }
            }
        }
        /** * 按条读 * 举例：abc\ndef * 分隔符可以是多个字符，如：abc\n\rdedf * 如果设置成\n，就是按行读了 * @param filePath 文件路径 * @param encoding 字符集 * @param bufferSize 每次读多少字节 * @param itemSplit 第条的分隔符 */
        public static void readByItem(String filePath,String encoding,int bufferSize,String itemSplit) {
            RandomAccessFile raf = null;
            FileChannel fc = null;
            try {
                raf = new RandomAccessFile(filePath, "r");
                fc = raf.getChannel();
                ByteBuffer buffer = ByteBuffer.allocate(bufferSize);
                byte[] itemSplitByte = itemSplit.getBytes(encoding);
                int itemSplitLen = itemSplitByte.length;
                //保存未处理的数据
                byte[] temp = new byte[0];
                int itemIndex = 0;
                while (fc.read(buffer)!= -1) {
                    int position = buffer.position();// 读取结束后的位置，相当于读取的长度
                    byte[] bs = new byte[position];// 用来存放读取的内容的数组
                    buffer.rewind();// 将position设回0,所以你可以重读Buffer中的所有数据,此处如果不设置,无法使用下面的get方法
                    buffer.get(bs);// 相当于buffer.get(bs,0,bs.length())：从position初始位置开始相对读,读bs.length个byte,并写入bs[0]到bs[bs.length-1]的区域
                    buffer.clear();
                    //要加上上次未处理的数据
                    int readSize = position + temp.length;
                    byte[] readByte = new byte[readSize];
                    System.arraycopy(temp, 0, readByte, 0, temp.length);
                    System.arraycopy(bs, 0, readByte, temp.length, position);
                    int start = 0;
                    int columnByteSize = 0;
                    byte[] columnByte = new byte[0];
                    int itemSplitIndex = 0;
                    for (int i = 0; i < readSize; i++) {
                        columnByteSize++;
                        if (readByte[i] == itemSplitByte[itemSplitIndex]) {
                            itemSplitIndex++;
                            if(itemSplitIndex==itemSplitLen){
                                columnByteSize = columnByteSize-itemSplitLen;
                                columnByte = new byte[columnByteSize];
                                System.arraycopy(readByte, start, columnByte, 0, columnByteSize);
                                String column = new String(columnByte, encoding);
                                System.out.println(itemIndex+"\t"+column);
                                itemIndex++;
                                start = i + 1;
                                columnByteSize = 0;
                                itemSplitIndex = 0;
                            }
                        } else {
                            itemSplitIndex=0;
                        }
    
                    }
                    // 把剩余的放入临时数组中
                    temp = new byte[columnByteSize];
                    System.arraycopy(readByte, start, temp, 0, columnByteSize);
                }
                //处理临时数组中的数据
                System.out.println("temp.length="+temp.length);
                if(temp.length>0){
                    String column = new String(temp, encoding);
                    System.out.println(itemIndex+"\t"+column);
                }
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                if (fc != null) {
                    try {
                        fc.close();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                }
                if (raf != null) {
                    try {
                        raf.close();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                }
            }
        }
        /** * 按条读，每一条又有被分成若干个列 * 举例：a|b|\nc|d\n * 分隔符可以是多个字符，如：a::b::c\n\rd::e::f\n\r * 严重声明：条分隔符与列分隔符不能有一样的符号，如columnSplit=|,itemSplit=||，它们都有|符号。 * 因为这个是逐个读的，不太好区分读到||的第一个字符时要不要切分列 * @param filePath 文件路径 * @param encoding 字符集 * @param bufferSize 每次读多少字节 * @param itemSplit 第条的分隔符 * @param columnSplit 每条中每列的分隔符 */
        public static void readByItem(String filePath,String encoding,int bufferSize,String itemSplit,String columnSplit) {
            RandomAccessFile raf = null;
            FileChannel fc = null;
            try {
                raf = new RandomAccessFile(filePath, "r");
                fc = raf.getChannel();
                ByteBuffer buffer = ByteBuffer.allocate(bufferSize);
                byte[] columnSplitByte = columnSplit.getBytes(encoding);
                int columnSplitLen = columnSplitByte.length;
                byte[] itemSplitByte = itemSplit.getBytes(encoding);
                int itemSplitLen = itemSplitByte.length;
                //保存未处理的数据
                byte[] temp = new byte[0];
                int columnIndex = 0;
                int itemIndex = 0;
                while (fc.read(buffer)!= -1) {
                    int position = buffer.position();// 读取结束后的位置，相当于读取的长度
                    byte[] bs = new byte[position];// 用来存放读取的内容的数组
                    buffer.rewind();// 将position设回0,所以你可以重读Buffer中的所有数据,此处如果不设置,无法使用下面的get方法
                    buffer.get(bs);// 相当于buffer.get(bs,0,bs.length())：从position初始位置开始相对读,读bs.length个byte,并写入bs[0]到bs[bs.length-1]的区域
                    buffer.clear();
                    //要加上上次未处理的数据
                    int readSize = position + temp.length;
                    byte[] readByte = new byte[readSize];
                    System.arraycopy(temp, 0, readByte, 0, temp.length);
                    System.arraycopy(bs, 0, readByte, temp.length, position);
                    int start = 0;
                    int columnByteSize = 0;
                    byte[] columnByte = new byte[0];
                    int columnSplitIndex = 0;
                    int itemSplitIndex = 0;
                    for (int i = 0; i < readSize; i++) {
                        columnByteSize++;
                        if (readByte[i] == columnSplitByte[columnSplitIndex]) {
                            columnSplitIndex++;
                            if(columnSplitIndex==columnSplitLen){
                                columnByteSize = columnByteSize-columnSplitLen;
                                columnByte = new byte[columnByteSize];
                                System.arraycopy(readByte, start, columnByte, 0, columnByteSize);
                                String column = new String(columnByte, encoding);
                                System.out.println(columnIndex+"\t"+column);
                                columnIndex++;
                                start = i + 1;
                                columnByteSize = 0;
                                columnSplitIndex = 0;
                            }
                        } else {
                            columnSplitIndex = 0;
                        }
                        if (readByte[i] == itemSplitByte[itemSplitIndex]) {
                            itemSplitIndex++;
                            if(itemSplitIndex==itemSplitLen){
                                columnByteSize = columnByteSize-itemSplitLen;
                                columnByte = new byte[columnByteSize];
                                System.arraycopy(readByte, start, columnByte, 0, columnByteSize);
                                String column = new String(columnByte, encoding);
                                System.out.println(columnIndex+"\t"+column);
                                System.out.println(itemIndex+"==========");
                                columnIndex=0;
                                itemIndex++;
                                start = i + 1;
                                columnByteSize = 0;
                                itemSplitIndex = 0;
                            }
                        } else {
                            itemSplitIndex=0;
                        }
    
                    }
                    // 把剩余的放入临时数组中
                    temp = new byte[columnByteSize];
                    System.arraycopy(readByte, start, temp, 0, columnByteSize);
                }
                //处理临时数组中的数据
                System.out.println("temp.length="+temp.length);
                if(temp.length>0){
                    String column = new String(temp, encoding);
                    System.out.println(columnIndex+"\t"+column);
                }
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                if (fc != null) {
                    try {
                        fc.close();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                }
                if (raf != null) {
                    try {
                        raf.close();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                }
            }
        }
    
        public static void main(String[] args) {
            long t1 = System.currentTimeMillis();
            String columnSplit = "|";
            String itemSplit = "\n";
            int bufferSize = 48;
            String encoding = "GBK";
            String filePath = "d:/data/a1.txt";
            //readByLine(filePath,encoding,bufferSize);
            //readByItem(filePath,encoding,bufferSize,itemSplit);
            readByItem(filePath,encoding,bufferSize,itemSplit,columnSplit);
            long t2 = System.currentTimeMillis();
            System.out.println("time==="+(t2-t1));
        }
    }

写文件

package com.syz.test.nio;
    
    import java.io.IOException;
    import java.io.RandomAccessFile;
    import java.nio.ByteBuffer;
    import java.nio.channels.FileChannel;
    import java.text.SimpleDateFormat;
    import java.util.Date;
    
    public class DataWriter {
        public static void main(String[] args) {
            long t1 = System.currentTimeMillis();
            write();
            long t2 = System.currentTimeMillis();
            System.out.println("time==="+(t2-t1));
        }
    
        public static void write() {
            SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
            String date = sdf.format(new Date());
            StringBuffer sb = new StringBuffer();
            int bufferSize = 1024;
            String encoding = "UTF-8";
            String filePath = "d:/data/a_"+date+".txt";
            RandomAccessFile raf = null;
            FileChannel fc = null;
            try {
                raf = new RandomAccessFile(filePath, "rws");
                fc = raf.getChannel();
                ByteBuffer buffer = ByteBuffer.allocate(bufferSize);
                int count = 0;
                for(int i=0;i<10;i++){ 
                    sb = new StringBuffer();
                    sb.append("20170410_"+i);
                    sb.append("|");
                    sb.append("127.0.0.1");
                    sb.append("|");
                    sb.append("测试中文");
                    sb.append("|");
                    sb.append("www.xxx.com"+date+"_"+i);
                    sb.append("|");
                    sb.append("");
                    sb.append("|");
                    sb.append("");
                    sb.append("|");
                    sb.append("测试中文222abc");
                    sb.append("|");
                    sb.append("测ha");
                    sb.append("|");
                    sb.append("");
                    sb.append("|");
                    sb.append("2017-03-02");
                    sb.append("|");
                    sb.append("2017-03-02");
                    sb.append("|");
                    sb.append("2017-03-02");
                    sb.append("|");
                    sb.append("22haa测试");
                    sb.append("|");
                    sb.append("abc123");
                    sb.append("\n");
                    String data = sb.toString();
                    byte[] dataByte = data.getBytes(encoding);
                    int dataLen = dataByte.length;
                    for(int j=0;j<dataLen;j++){ 
                        if(count==bufferSize){
                            buffer.flip();
                            int a  = fc.write(buffer);
                            System.out.println("a="+a);
                            buffer.clear();
                            count=0;
                        }
                        buffer.put(dataByte[j]);
                        count ++;
                    }
                }
                buffer.flip();
                //写入最后一点儿
                int b = fc.write(buffer);
                System.out.println("b="+b);
            } catch (Exception e) {
                e.printStackTrace();
            }finally{
                if(fc!=null){
                    try {
                        fc.close();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                }
                if(raf!=null){
                    try {
                        raf.close();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                }
            }
        }
    }