Java网络爬虫Spider

三更灯火五更鸡，正是男儿读书时。

 本次爬取百度贴吧的图片。

本次爬虫网址为  https://tieba.baidu.com/f?ie=utf-8&kw=%E9%A3%8E%E6%99%AF%E5%9B%BE&fr=search

 本次爬虫用到的工具： IJ URL类，IO流，UUID(随机命名)，正则表达式。

首先得到 https://tieba.baidu.com/f?ie=utf-8&kw=%E9%A3%8E%E6%99%AF%E5%9B%BE&fr=search   的html。

 HttpURLConnection connection =(HttpURLConnection)
                new URL("https://tieba.baidu.com/f?ie=utf-8&kw=%E9%A3%8E%E6%99%AF%E5%9B%BE&fr=search").openConnection();

返回 一个，字符串形式的html便于后面操作。

 接下来就要写正则来匹配图片的地址。首先要分析我们要的图片是那个地址。

我们要的是中间的风景图片，而不是其他的二维码图片或者头像图片，这个时候就打开开发者工具F12

 那么将有用的代码拿出来，供我们写正则

data-original="https://imgsa.baidu.com/forum/wh%3D200%2C90%3B/sign=561eeed4c4ea15ce41bbe80b863016ca/57335443fbf2b211c636051fc78065380cd78e56.jpg"  



bpic="https://imgsa.baidu.com/forum/w%3D580%3B/sign=8436c6918f025aafd3327ec3cbd6aa64/a08b87d6277f9e2f522d9d591230e924b899f343.jpg" 

 经过验证，上面的是我们需要的风景图片的地址，就是date-original后面的。

注意：正则写的时候，需要注意我们把后面的bpic带上来精确正则表达式。于是就开始匹配图片地址，放再一个List中。

 private static ArrayList getPictureUrl(String html) {
        Pattern patternRegex = Pattern.compile("bpic=\"(.*?)\"");//匹配图片地址的正则
        Matcher matcher = patternRegex.matcher(html);
        String htmlUrl=null;

        ArrayList<String> listUrl = new ArrayList<>();
        while (matcher.find()){
            htmlUrl = matcher.group();
            String[] split = htmlUrl.split("=\"");
            String[] split1 = split[1].split("\"");//切出来了标准的URL
            listUrl.add(split1[0]);//将图片的url存入数组中
        }
        return listUrl;
    }

 返回一个listURl，那么接下来就继续再listURL中去下载图片，用到IO流。

 private static void getPicture(ArrayList pictureUrl) throws IOException {
        for (Object p : pictureUrl) {
           HttpURLConnection conn = (HttpURLConnection) new URL((String)p).openConnection();
            InputStream in = conn.getInputStream();
            UUID uuid = UUID.randomUUID();//形成随机名字
            FileOutputStream out = new FileOutputStream("C:\\Users\\Administrator\\IdeaProjects\\homework\\picture\\"+uuid+".png");
            while (true){
                byte[] bytes = new byte[1024];
                int len = in.read(bytes);
                if(len == -1){
                    break;
                }
                out.write(bytes,0,len);
            }
            out.close();
        }
    }

 完整代码

package com.westos.danli;

import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.UUID;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class Spider1 {
    public static void main(String[] args) throws IOException {
        HttpURLConnection connection =(HttpURLConnection)
                new URL("https://tieba.baidu.com/f?ie=utf-8&kw=%E9%A3%8E%E6%99%AF%E5%9B%BE&fr=search").openConnection();
//        InputStream in = connection.getInputStream();
//        BufferedReader reader;
//        reader = new BufferedReader(new FileReader(String.valueOf(in)));
        String html = Spider1.getHtml(connection);//返回一个字符串的html对象
        ArrayList pictureUrl = getPictureUrl(html);
        getPicture(pictureUrl);//直接存在C:\Users\Administrator\IdeaProjects\homework\picture中

        return;
    }

    private static void getPicture(ArrayList pictureUrl) throws IOException {
        for (Object p : pictureUrl) {
           HttpURLConnection conn = (HttpURLConnection) new URL((String)p).openConnection();
            InputStream in = conn.getInputStream();
            UUID uuid = UUID.randomUUID();//形成一个随机命名。
            FileOutputStream out = new FileOutputStream("C:\\Users\\Administrator\\IdeaProjects\\homework\\picture\\"+uuid+".png");//直接用网址命名
            while (true){
                byte[] bytes = new byte[1024];
                int len = in.read(bytes);
                if(len == -1){
                    break;
                }
                out.write(bytes,0,len);
            }
            out.close();
        }
    }
    private static ArrayList getPictureUrl(String html) {
        Pattern patternRegex = Pattern.compile("bpic=\"(.*?)\"");//匹配图片地址的正则
        Matcher matcher = patternRegex.matcher(html);
        String htmlUrl=null;

        ArrayList<String> listUrl = new ArrayList<>();
        while (matcher.find()){
            htmlUrl = matcher.group();
            String[] split = htmlUrl.split("=\"");
            String[] split1 = split[1].split("\"");//切出来了标准的URL
            listUrl.add(split1[0]);//将图片的url存入数组中
        }
        return listUrl;
    }
    /*
          获得html页面，写入本地。
     */
    private static String getHtml(HttpURLConnection connection) throws IOException {
        InputStream in = connection.getInputStream();//拿到html字节流
        BufferedReader reader = new BufferedReader(new InputStreamReader(in));
        BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(new FileOutputStream("123.html"));
        StringBuffer stringBuffer1 = new StringBuffer();//定义一个stringbuffer,将html转换为字符串对象
        while (true) {
            String s = reader.readLine();//读取html
            if (s == null) {
                break;
            }
            bufferedOutputStream.write(s.getBytes());
            stringBuffer1.append(s).append("\n");//存为字符串对象
        }
        return stringBuffer1.toString();
    }


    }