【Python】小说爬取,源可能失效

wangy

1. from time import time
   " B( C* E7 Z: T, p8 c+ \( Q2 M
2. import requests
   
3. import re
   
4. import os
   
5. from multiprocessing.dummy import Pool
   & \* d% Z9 v7 x8 m3 o% {
6. 
   ; z  _% B# l- [
7. start_url = 'https://www.kanunu8.com/book4/10365/'
     K2 Q1 u& P, F! |- x' _/ e) q
8. 
   $ _  o0 z$ x+ Z
9. def get_source(url):
   
10.     html = requests.get(url)
    
11.     return html.content.decode('gbk')
    
12. 
    
13. def get_toc(html):
    + q9 x& x- M, T' a
14. 
    
15.     toc_url_list = []
    6 p4 y1 n' P% o0 n& o5 z1 g
16.     toc_block = re.findall('正文(.*?)</tbody>',html,re.S)[0]
    * I' `$ I& R- a! y% G$ I* P' I
17.     toc_url = re.findall('href="(.*?)"',toc_block,re.S)
    8 r( I- ~" u$ w( M% h+ F4 ?
18.     for url in toc_url:
    
19.         toc_url_list.append(start_url+url)
    
20.     return toc_url_list
    $ D6 N0 _7 |; W6 k5 O. ]/ M7 j
21. 
    4 B, e2 ]) g2 S% u6 F( Y" n, \' D  K
22. def get_article(html):
    3 s; Z$ `) X- k* p* S, ^
23. 
    
24.     chapter_name = re.search('size="4">(.*?)<',html,re.S).group(1)
    
25.     text_block = re.search('<p>(.*?)</p>',html,re.S).group(1)
    
26.     text_block = text_block.replace('<br />','')
    0 u8 x6 X& p* E! `$ Z) f& w9 p
27.     text_block = text_block.replace('&nbsp;','')
    
28.     return chapter_name,text_block
    1 g8 ]# {' L3 K6 J% _
29. 
    - C; }2 V" D4 R( S- e- Q" q7 f
30. def save(chapter,article):
    
31.     os.makedirs('罗密欧与朱丽叶',exist_ok=True)
    
32.     with open(os.path.join('罗密欧与朱丽叶',chapter + '.txt'),'w',encoding='utf-8') as f:
    7 E$ M# ^+ b8 A! s) l
33.         f.write(article)
    ) V3 x/ L- B3 K  C( s* \
34.         print('下载成功--->',chapter, '.txt')
    ' s6 S" O, \0 ^% u, s$ X
35. 
    ! q7 W, \% q( b$ W3 k
36. def query_article(url):
    6 q3 W+ k1 g7 ?3 ~
37.     article_html = get_source(url)
    
38.     chapter_name,article_text = get_article(article_html)
    
39.     save(chapter_name,article_text)
    
40. 
    ' ~) V4 s( ~. i, S2 Z  t
41. if __name__ == '__main__':
    ( y, ?1 l8 J3 p6 t# A
42.     a = get_source(start_url)
    
43.     b = get_toc(a)
    
44.     start = time()
    
45.     pool = Pool(4)
    & E2 r9 ?; G- t
46.     pool.map(query_article, b)
    
47.     end = time()
    ' X1 h$ N- {4 n' @0 o. G; |& n5 H
48.     print(end-start)
    # Q% P- h% G0 X& d

复制代码