使用python爬虫爬取迅雷侠下载，呵呵，你懂得

啥也不说了，直接上代码

#!python3
import urllib.request
from lxml import etree
import threading
from multiprocessing import Pool
from multiprocessing.dummy import Pool as ThreadPool


#龙图腾原创

#请求  
#三种浏览器的user-agent
#headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.63 Safari/537.36'}
#headers  = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11'}
headers  = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0'}


max_try_down = 20
max_timeout_down = 1
allresult = ""      #最终要下载的内容
allresult_count = 1; #下载个数


all_page_item = [];  


#将下载结果总结到allresult
lock = threading.Lock();#线程锁初始化
def add_inresult(weburl,name,link,size):
    global allresult_count,allresult


    lock.acquire();#线程锁加锁
    if link not in allresult:
        allresult_count = allresult_count+1;
        allresult += ("%s\r\n"%(weburl));
        #allresult += ("len name=%d link=%d %d\r\n"%(len(name),len(link),len(size)));
        allresult += ("name=%s\r\n"%(name));
        allresult += ("size=%s\r\n"%(size));
        allresult += ("link=%s\r\n"%(link));
        allresult += ("\r\n");
    lock.release();#线程锁解锁






#下载文件
def download_this(weburl):


    #print("down %s++\n"%(weburl),end='');#因为使用了threadpool故此打印\n可能会遇到问题，故此使用这种打印方法
   
    #下载循环尝试4次
    for i in range(max_try_down):
        try:
            sub_req = urllib.request.Request(url=weburl,headers=headers)  
            sub_response = urllib.request.urlopen(sub_req,timeout=max_timeout_down)
            sub_data = sub_response.read()
            sub_data = sub_data.decode('utf-8')
            tree=etree.HTML(sub_data);


           
            file_name = " ".join(tree.xpath(u"//h3/text()"));#找到h3的text
            file_size = tree.xpath(u"//b[text()='文件大小']/following-sibling::span")[0].text;#找到文本="文件大小"的节点 following-sibling::span寻找下一个span节点
            file_link = tree.xpath(u"//a[@href=*]/@href")[0];#寻找存在href的 a节点，并且提取href
            add_inresult(weburl,file_name,file_link,file_size);
            break
        except IndexError as e:
            add_inresult(weburl,file_name,"连接失效:"+weburl,file_size);
        except Exception as e:
            print("%s [download_this]重试 %d <%s>:%s\n"%(weburl,i,type(e),e),end='');
            pass
        
    print("down %s ok--\n"%(weburl),end='');#因为使用了threadpool故此打印\n可能会遇到问题，故此使用这种打印方法




#列出当前页面的下载列表
def list_this(weburl):
    global all_page_item
    for i in range(max_try_down):
        try:
            #print("+++search+++%s\n"%weburl,end='');#因为使用了threadpool故此打印\n可能会遇到问题，故此使用这种打印方法


            request = urllib.request.Request(url=weburl,headers=headers)    
            #爬取结果  
            response = urllib.request.urlopen(request,timeout=max_timeout_down)  


            data = response.read()




            
            #设置解码方式  
            data = data.decode('utf-8')


            tree=etree.HTML(data);


            nodes = tree.xpath(u"//h5[@class='item-title']/a/@href");#找寻h5节点的class属性为item-title下的a节点/提取href属性


            if len(nodes) <= 0:
                return False


            all_page_item += nodes;#统计所有的页面下的下载页面


            
            print("allresult_count=%d"%len(all_page_item));
            print("---search ok---%s \n"%weburl,end='');#因为使用了threadpool故此打印\n可能会遇到问题，故此使用这种打印方法
            return True;
        except Exception as e:
            print("%s [list_this]重试 %d <%s>:%s\n"%(weburl,i,type(e),e),end='');#因为使用了threadpool故此打印\n可能会遇到问题，故此使用这种打印方法
            pass




def search_this(name):




    while True:
        try:
            url = "https://www.soxunlei.com/search/%s/page/%d.html"%(urllib.request.quote(name),1)
            request = urllib.request.Request(url=url,headers=headers)    
            #爬取结果  
            response = urllib.request.urlopen(request,timeout=1)  


            data = response.read()




            #设置解码方式  
            data = data.decode('utf-8')


            tree=etree.HTML(data);


            count = int(tree.xpath(u"//ol[@class='breadcrumb']/li[4]/font[2]")[0].text);#搜索返回个数，并且算出有多少页
            page_count = count / 15 + 1  #一页容纳15个下载页，算出页数
            print("%s搜索到%d个内容，分为%d页"%(name,count,page_count))




            #讲下载页的url算出放入page_nodes
            page_nodes = [];
            for n in range(1,int(page_count)+1):
                url = "https://www.soxunlei.com/search/%s/page/%d.html"%(urllib.request.quote(name),n)
                page_nodes.append(url);
                print("url=%s"%url);
            
            break
        except Exception as e:
            print("%s:%s"%(type(e),e));
    #调用python线程池模型，抓取所有下载页中的下载项
    pool = ThreadPool(32)
    results = pool.map(list_this,page_nodes);
    pool.close();
    pool.join();  


    #线程池模型,下载所有内容
    pool = ThreadPool(32)
    results = pool.map(download_this,all_page_item);
    pool.close();
    pool.join();


    
    print(allresult);
    print("抓取完成 项%d 下载%d"%(count,allresult_count));
if __name__ == "__main__":
    search_this("上原空");