Skip to content

Latest commit

 

History

History
118 lines (93 loc) · 3.9 KB

68.存储数据.md

File metadata and controls

118 lines (93 loc) · 3.9 KB
Raw

存储数据

存储海量数据

数据持久化的首选方案应该是关系型数据库，关系型数据库的产品很多，包括：Oracle、MySQL、SQLServer、PostgreSQL等。如果要存储海量的低价值数据，文档数据库也是不错的选择，MongoDB是文档数据库中的佼佼者，有兴趣的读者可以自行研究。

下面的代码演示了如何使用MySQL来保存从知乎发现上爬取到的链接和页面。

create database zhihu default charset utf8;
create user 'hellokitty'@'%' identified by 'Hellokitty.618';
grant all privileges on zhihu.* to 'hellokitty'@'%';
flush privileges;

use zhihu;
create table `tb_explore`
(
	`id` integer auto_increment,
    `url` varchar(1024) not null,
    `page` longblob not null,
    `digest` char(48) unique not null,
    `idate` datetime default now(),
    primary key (`id`)
);
import hashlib
import pickle
import re
import zlib
from urllib.parse import urljoin

import MySQLdb
import bs4
import requests

conn = MySQLdb.connect(host='1.2.3.4', port=3306,
                       user='hellokitty', password='Hellokitty.618',
                       database='zhihu', charset='utf8',
                       autocommit=True)


def write_to_db(url, page, digest):
    try:
        with conn.cursor() as cursor:
            cursor.execute(
                'insert into tb_explore (url, page, digest) values (%s, %s, %s) ',
                (url, page, digest)
            )
    except MySQLdb.MySQLError as err:
        print(err)


def main():
    base_url = 'https://www.zhihu.com/'
    seed_url = urljoin(base_url, 'explore')
    headers = {'user-agent': 'Baiduspider'}
    try:
        resp = requests.get(seed_url, headers=headers)
        soup = bs4.BeautifulSoup(resp.text, 'lxml')
        href_regex = re.compile(r'^/question')
        for a_tag in soup.find_all('a', {'href': href_regex}):
            href = a_tag.attrs['href']
            full_url = urljoin(base_url, href)
            digest = hashlib.sha1(full_url.encode()).hexdigest()
            html_page = requests.get(full_url, headers=headers).text
            zipped_page = zlib.compress(pickle.dumps(html_page))
            write_to_db(full_url, zipped_page, digest)
    finally:
        conn.close()


if __name__ == '__main__':
    main()

数据缓存

通过《网络数据采集和解析》一文，我们已经知道了如何从指定的页面中抓取数据，以及如何保存抓取的结果，但是我们没有考虑过这么一种情况，就是我们可能需要从已经抓取过的页面中提取出更多的数据，重新去下载这些页面对于规模不大的网站倒是问题也不大，但是如果能够把这些页面缓存起来，对应用的性能会有明显的改善。下面的例子演示了如何使用Redis来缓存知乎发现上的页面。

import hashlib
import pickle
import re
import zlib
from urllib.parse import urljoin

import bs4
import redis
import requests


def main():
    base_url = 'https://www.zhihu.com/'
    seed_url = urljoin(base_url, 'explore')
    client = redis.Redis(host='1.2.3.4', port=6379, password='1qaz2wsx')
    headers = {'user-agent': 'Baiduspider'}
    resp = requests.get(seed_url, headers=headers)
    soup = bs4.BeautifulSoup(resp.text, 'lxml')
    href_regex = re.compile(r'^/question')
    for a_tag in soup.find_all('a', {'href': href_regex}):
        href = a_tag.attrs['href']
        full_url = urljoin(base_url, href)
        field_key = hashlib.sha1(full_url.encode()).hexdigest()
        if not client.hexists('spider:zhihu:explore', field_key):
            html_page = requests.get(full_url, headers=headers).text
            zipped_page = zlib.compress(pickle.dumps(html_page))
            client.hset('spider:zhihu:explore', field_key, zipped_page)
    print('Total %d question pages found.' % client.hlen('spider:zhihu:explore'))


if __name__ == '__main__':
    main()