webCrawler-/qiushi.py at master · devinyf/webCrawler- · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
import requests
from lxml import etree
import time
import json


class Qiushi(object):
    def __init__(self):
        self.base_url = 'https://www.qiushibaike.com/8hr/page/%s/'
        self.url_list = None
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.89 Safari/537.36'
        }
        self.file = open('day03/qiushi.json', 'w', encoding='utf8')

    def generate_url(self):
        self.url_list = [self.base_url % (i) for i in range(1, 14)]
        # print(self.url_list)

    def get_page(self, url):
        response = requests.get(url, headers=self.headers)
        return response.content

    def parse_data(self, data):
        html = etree.HTML(data)

        node_list = html.xpath('//div[@id="content-left"]/div')
        # print(len(node_list))

        data_list = []
        for node in node_list:
            temp = {}
            try:
                temp['user'] = node.xpath('./div[1]/a[2]/h2/text()')[0].strip()
                temp['zone_link'] = 'https://www.qiushibaike.com/' + \
                    node.xpath('./div[1]/a[2]/@href')[0]
                temp['age'] = node.xpath('./div[1]/div/text()')[0]
                temp['gender'] = node.xpath(
                    './div[1]/div/@class')[0].split(' ')[-1].replace('Icon', '')
            except:
                temp['user'] = '匿名用户'
                temp['zone_link'] = None
                temp['age'] = None
                temp['gender'] = None

            temp['content'] = ''.join(node.xpath(
                './a[1]/div/span[1]/text()')).strip()
            # print(temp)
            data_list.append(temp)
        return data_list

    def save_data(self, data_list):
        for data in data_list:
            str_data = json.dumps(data, ensure_ascii=False) + ',\n'
            self.file.write(str_data)

    def __del__(self):
        self.file.close()

    def run(self):
        self.generate_url()

        for url in self.url_list:
            data = self.get_page(url)
            data_list = self.parse_data(data)
            time.sleep(1)
            self.save_data(data_list)


if __name__ == '__main__':
    qiushi = Qiushi()
    qiushi.run()