用java获取虎扑新闻数据

我采用多线程进行任务抓取  也可以采用单线程

public void begin() {

                //获取虎扑新闻的数据，网址格式为：https://voice.hupu.com/nba/第几页   

            new Thread(new Runnable() {

                @Override

                public void run() {

                    try{


 


                    Document doc = Jsoup.connect(“https://voice.hupu.com/nba/”).get();

                    Elements titleLinks = doc.select(“div.list-hd”);    //解析来获取每条新闻的标题与链接地址

          

                    Elements timeLinks = doc.select(“div.otherInfo”);   //解析来获取每条新闻的时间与来源

                    

                    //for循环遍历获取到每条新闻的四个数据并封装到News实体类中

                    for(int j = 0;j < titleLinks.size();j++){

                        String title = titleLinks.get(j).select(“a”).text();

                        String uri = titleLinks.get(j).select(“a”).attr(“href”);

                        Document doc1 = Jsoup.connect(uri).get();

                        Elements desLinks=doc1.select(“div.artical-main-content”);

                        String desc=””;

                       if(desLinks.size()!=0){

                           Elements e=desLinks.select(“p”);

                           for(int z = 0;z < e.size();z++){

                               desc+=e.get(z).text();

                           }

                       }else{

                           desc=”0″;

                       }

                        

                        String time = timeLinks.get(j).select(“span.other-left”).select(“a”).text();

                        News news = new News(title,uri,desc,time);

                        String sql =”insert into bll_pageinfo values (null,?,?,?,?)”;

                        String[] parameters = { news.getNewsTitle(), news.getNewsUrl(), news.getDesc(), news.getNewsTime()};

                        DbAction.executeUpdate(sql,parameters );

                        System.out.println(“当前的线程为：”+Thread.currentThread().getName());

                    }

                }

                

            }catch (Exception e){

               e.printStackTrace();

            }

       }

   }).start();

 }    

获取的数据保存到数据库  自己数据库可以配置字段跟实体类对应

我的数据库是mysql，小伙伴也可以根据自己的来选择

public class DbAction {

public static int executeUpdate(String sql, String[] parameters) {

        int result = 0;

        Connection conn = null;

        PreparedStatement pstmt = null;

        try {

            conn = getConnection();

            pstmt = conn.prepareStatement(sql);

            for (int i = 0; i < parameters.length; i++) {

                pstmt.setString(i + 1, parameters[i]);

            }

            result = pstmt.executeUpdate();

        } catch (SQLException err) {

            err.printStackTrace();

            free(null, pstmt, conn);

        } finally {

            free(null, pstmt, conn);

        }

        return result;

    }       

 }   

        

        

实体类封装抓取的数据

public class News {

    
        private String newsTitle;   //新闻标题
        private String newsUrl;     //新闻链接地址
        private String desc;        //新闻概要
        private String newsTime;    //新闻时间与来源

        public News(String newsTitle, String newsUrl, String desc, String newsTime) {

            this.newsTitle = newsTitle;
            this.newsUrl = newsUrl;
            this.desc = desc;
            this.newsTime = newsTime;
        }

        public String getDesc() {

            return desc;
        }

        public void setDesc(String desc) {

            this.desc = desc;
        }

        public String getNewsTime() {

            return newsTime;
        }

        public void setNewsTime(String newsTime) {

            this.newsTime = newsTime;
        }

        public String getNewsTitle() {

            return newsTitle;
        }

        public void setNewsTitle(String newsTitle) {

            this.newsTitle = newsTitle;
        }

        public String getNewsUrl() {

            return newsUrl;
        }

        public void setNewsUrl(String newsUrl) {

            this.newsUrl = newsUrl;
        }
    }

  个人亲测有效

需要的jar包

本人JDK使用的1.8