java实现html的文本提取-Demo

超、凢脫俗 2022-09-11 01:13 163阅读 0赞

package com.ydfind.util;
    
    import com.alibaba.druid.util.StringUtils;
    import org.jsoup.Jsoup;
    import org.jsoup.helper.StringUtil;
    import org.jsoup.nodes.Element;
    import org.jsoup.nodes.Node;
    import org.jsoup.nodes.TextNode;
    import org.jsoup.select.NodeTraversor;
    import org.jsoup.select.NodeVisitor;
    
    import java.util.ArrayList;
    import java.util.List;
    
    /**
     * 需要加maven依赖
     *
     <dependency>
     <groupId>org.jsoup</groupId>
     <artifactId>jsoup</artifactId>
     <version>1.11.3</version>
     </dependency>
     */
    public class HtmlUtils {
        public static String wholeText(String html, String delimiter) {
            return String.join(delimiter, wholeText(Jsoup.parse(html)));
        }
    
        public static List<String> wholeText(Element root) {
            List<String> texts = new ArrayList<>();
            NodeTraversor.traverse(new NodeVisitor() {
                @Override
                public void head(Node node, int i) {
                    if (node instanceof TextNode) {
                        String text = ((TextNode) node).getWholeText().trim();
                        text = trimHtmlText(text);
                        if (!StringUtils.isEmpty(text)) {
                            texts.add(text);
                        }
                    }
                }
    
                @Override
                public void tail(Node node, int i) {
    
                }
            }, root);
            return texts;
        }
    
        private static String trimHtmlText(String text) {
            if (text.length() == 0) {
                return text;
            }
            // 去掉两边空格和不可见字符
            int id = 0;
            for (; id < text.length(); id++) {
                if (!isWhitespaceOrInvisibleChar(text.codePointAt(id))){
                    break;
                }
            }
            int start = id;
            for (id = text.length() - 1; id > start; id--) {
                if (!isWhitespaceOrInvisibleChar(text.codePointAt(id))){
                    break;
                }
            }
            int end = id;
            int c;
            boolean lastSpace = false;
            StringBuilder sb = new StringBuilder();
            for (id = start; id <= end; id++) {
                c = text.codePointAt(id);
                if (isWhitespaceOrInvisibleChar(c)) {
                    if (!lastSpace) {
                        lastSpace = true;
                        sb.append(' ');
                    }
                } else if (!StringUtil.isInvisibleChar(c)) {
                    sb.appendCodePoint(c);
                    lastSpace = false;
                }
            }
            return sb.toString();
        }
    
        private static boolean isWhitespaceOrInvisibleChar(int c) {
            return isWhitespace(c) || StringUtil.isInvisibleChar(c);
        }
    
        private static boolean isWhitespace(int c) {
            // 12288为中文空格
            return StringUtil.isActuallyWhitespace(c) || c == 12288;
        }
    }

提取的文本可以提供算法使用