抓取百度关键词排名、标题、连接、描述

抓取百度关键词排名、标题、连接、描述

转载请标明出处

最近在做百度关键词排名的功能，发现网上资源比较这里写代码片少，于是自己琢磨了一下，写一下笔记；
本文重点在于提供思路，请不要过分依赖，本文主要靠抓取页面标签来完成，如果百度官网将页面标签修改了，请自行修改，如果遇到问题或需要修改的地方请私信我。
鸣谢：本公司SEO提供思路

package cc.test.core;


import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.Tag;
import org.htmlparser.filters.AndFilter;
import org.htmlparser.filters.HasAttributeFilter;
import org.htmlparser.filters.HasParentFilter;
import org.htmlparser.filters.NodeClassFilter;
import org.htmlparser.filters.OrFilter;
import org.htmlparser.filters.TagNameFilter;
import org.htmlparser.tags.Div;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.tags.TitleTag;
import org.htmlparser.util.NodeList;


public class KeywordRun {
    /**
     * 
     * @Description: 获取排名数
     * @param @param keyword-关键词
     * @param @param url-域名
     * @param @return   
     * @return int  
     */
    public int getKeywordRank(String keyword, String url) {
        int re = 0;
        // for (int i = 0; i < 100; i += 10) {
        // if (i == 0) {
        re = getThisRank("http://www.baidu.com/s?wd=" + keyword, url);
        //
        // if (re > 0)
        // break;
        // } else {
        // re = getThisRank("http://www.baidu.com/s?wd=" + keyword + "&pn=" + i,
        // url);
        // if (re > 0) {
        // re += i;
        // break;
        // }
        // }
        // }
        return re;
    }

    /**/
    public int getThisRank(String resource, String url) {
        int re = -1;
        int n = 1;
        try {
            Parser myParser = new Parser(resource);

            // 设置编码
            myParser.setEncoding("UTF-8");

            NodeFilter filter = new AndFilter(new TagNameFilter("DIV"), new HasParentFilter(new AndFilter(new TagNameFilter("DIV"), new HasAttributeFilter("id", "content_left"))));
            NodeList nodeList = myParser.extractAllNodesThatMatch(filter);

            for (int i = 0; i < nodeList.size(); i++) {

                Div table = (Div) nodeList.elementAt(i);

                Parser parser = new Parser(table.toHtml());

                NodeFilter TitleFilter = new NodeClassFilter(TitleTag.class);
                NodeFilter ElementIdFilter = new HasAttributeFilter("class", "c-showurl");// 获取链接
                OrFilter orFilter = new OrFilter(TitleFilter, ElementIdFilter); // 做一个逻辑OR
                                                                                // Filter组合
                NodeList list = parser.extractAllNodesThatMatch(orFilter);

                LinkTag linkTag = (LinkTag) list.elementAt(0);

                String link = HttpUtil.getBaiduFinalLink(linkTag.getLink());// 最终的连接
                System.out.println(link);

                parser = new Parser(table.toHtml());
                TitleFilter = new NodeClassFilter(TitleTag.class);
                ElementIdFilter = new HasAttributeFilter("class", "t");// 获取标题
                orFilter = new OrFilter(TitleFilter, ElementIdFilter); // 做一个逻辑OR
                                                                        // Filter组合
                list = parser.extractAllNodesThatMatch(orFilter);

                Tag tag = (Tag) list.elementAt(0);
                linkTag = (LinkTag) tag.getChildren().elementAt(0);

                //System.out.println(splitAndFilterString(linkTag.getChildrenHTML(),linkTag.getChildrenHTML().length()));

                parser = new Parser(table.toHtml());
                TitleFilter = new NodeClassFilter(TitleTag.class);
                ElementIdFilter = new HasAttributeFilter("class", "c-abstract");// 获取标题
                orFilter = new OrFilter(TitleFilter, ElementIdFilter); // 做一个逻辑OR
                // Filter组合
                list = parser.extractAllNodesThatMatch(orFilter);

                Div div = (Div) list.elementAt(0);
                String _abstract = splitAndFilterString(div.toHtml(),div.toHtml().length());//简介

                System.out.println(_abstract);


            }

            // System.out.println(text);
        } catch (Exception e) {
            e.printStackTrace();
            re = -1;
        }
        return re;
    }
/**
     * 
     * @Description: 调试
     * @param @param args   
     * @return void  
     * @throws
     */
    public static void main(String[] args) {

        KeywordRun run = new KeywordRun();

        Integer re = run.getKeywordRank("百度", "www.baidu.com");
    }
}

需要用到的外部方法

/**
     * 
     * 获取百度最终连接
     */
    public static String getBaiduFinalLink(String link){

        BufferedReader in = null;
        try {

            URL realUrl = new URL(link);
            // 打开和URL之间的连接
            HttpURLConnection connection = (HttpURLConnection)realUrl.openConnection();
            // 设置通用的请求属性
            connection.setRequestProperty("accept", "*/*");
            connection.setRequestProperty("connection", "Keep-Alive");
            connection.setRequestProperty("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;SV1)");
            // 建立实际的连接
            connection.connect();
            in = new BufferedReader(new InputStreamReader(connection.getInputStream(), "utf-8"));
            URL host = connection.getURL();
            if(connection.getResponseCode() >= 400){
                return null;
            }

            if (in != null) {
                in.close();
            }
            return host.getHost()+host.getPath();
        } catch (Exception e) {
            System.out.println("发送GET请求出现异常！" + e);
            e.printStackTrace();
        }

        return null;
    }
    /**
     * 删除input字符串中的html格式
     * 
     * @param input
     * @param length
     * @return
     */
    public static String splitAndFilterString(String input, int length) {
        if (input == null || input.trim().equals("")) {
            return "";
        }
        // 去掉所有html元素,
        String str = input.replaceAll("\\&[a-zA-Z]{1,10};", "").replaceAll("<[^>]*>", "");
        str = str.replaceAll("[(/>)<]", "");
        int len = str.length();
        if (len <= length) {
            return str;
        } else {
            str = str.substring(0, length);
            str += "......";
        }
        return str;
    }

到此结束，
已经获取出百度第一页的所有的连接、标题、简介，如果百度页面有修改，请做相对应的修改即可。

转载请标明出处