爬取小說網(wǎng)站的小說，并保存到數(shù)據(jù)庫
專注于為中小企業(yè)提供網(wǎng)站設(shè)計、做網(wǎng)站服務(wù),電腦端+手機端+微信端的三站合一,更高效的管理,為中小企業(yè)宜昌免費做網(wǎng)站提供優(yōu)質(zhì)的服務(wù)。我們立足成都，凝聚了一批互聯(lián)網(wǎng)行業(yè)人才，有力地推動了上千企業(yè)的穩(wěn)健成長，幫助中小企業(yè)通過網(wǎng)站建設(shè)實現(xiàn)規(guī)模擴充和轉(zhuǎn)變。
第一步：先獲取小說內(nèi)容
重慶分公司
重慶分公司
#!/usr/bin/python
# -*- coding: UTF-8 -*-
import urllib2,re
domain = 'http://www.quanshu.net'
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
}
def getTypeList(pn=1): #獲取分類列表的函數(shù)
req = urllib2.Request('http://www.quanshu.net/map/%s.html' % pn) #實例將要請求的對象
req.headers = headers #替換所有頭信息
#req.add_header() #添加單個頭信息
res = urllib2.urlopen(req) #開始請求
html = res.read().decode('gbk') #decode解碼，解碼成Unicode
reg = r'(.*?)'
reg = re.compile(reg) #增加匹配效率 正則匹配返回的類型為List
return re.findall(reg,html)
def getNovelList(url): #獲取章節(jié)列表函數(shù)
req = urllib2.Request(domain + url)
req.headers = headers
res = urllib2.urlopen(req)
html = res.read().decode('gbk')
reg = r'(.*?) style6\(\)'
return re.findall(reg,html)[0]
if __name__ == '__main__':
for type in range(1,10):
for url,title in getTypeList(type):
for zurl,ztitle in getNovelList(url):
print u'正則爬取----%s' %ztitle
content = getNovelContent(url.replace('index.html',zurl))
print content
break
break 執(zhí)行后結(jié)果如下：
第二步：存儲到數(shù)據(jù)庫
1、設(shè)計數(shù)據(jù)庫
1.1 新建庫：novel
1.2 設(shè)計表：novel
1.3 設(shè)計表：chapter
并設(shè)置外鍵
2、編寫腳本
#!/usr/bin/python
# -*- coding: UTF-8 -*-
import urllib2,re
import MySQLdb
class Sql(object):
conn = MySQLdb.connect(host='192.168.19.213',port=3306,user='root',passwd='Admin123',db='novel',charset='utf8')
def addnovels(self,sort,novelname):
cur = self.conn.cursor()
cur.execute("insert into novel(sort,novelname) values(%s , '%s')" %(sort,novelname))
lastrowid = cur.lastrowid
cur.close()
self.conn.commit()
return lastrowid
def addchapters(self,novelid,chaptername,content):
cur = self.conn.cursor()
cur.execute("insert into chapter(novelid,chaptername,content) values(%s , '%s' ,'%s')" %(novelid,chaptername,content))
cur.close()
self.conn.commit()
domain = 'http://www.quanshu.net'
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
}
def getTypeList(pn=1): #獲取分類列表的函數(shù)
req = urllib2.Request('http://www.quanshu.net/map/%s.html' % pn) #實例將要請求的對象
req.headers = headers #替換所有頭信息
#req.add_header() #添加單個頭信息
res = urllib2.urlopen(req) #開始請求
html = res.read().decode('gbk') #decode解碼，解碼成Unicode
reg = r'(.*?)'
reg = re.compile(reg) #增加匹配效率 正則匹配返回的類型為List
return re.findall(reg,html)
def getNovelList(url): #獲取章節(jié)列表函數(shù)
req = urllib2.Request(domain + url)
req.headers = headers
res = urllib2.urlopen(req)
html = res.read().decode('gbk')
reg = r'(.*?) style6\(\)'
return re.findall(reg,html)[0]
mysql = Sql()
if __name__ == '__main__':
for sort in range(1,10):
for url,title in getTypeList(sort):
lastrowid = mysql.addnovels(sort, title)
for zurl,ztitle in getNovelList(url):
print u'正則爬取----%s' %ztitle
content = getNovelContent(url.replace('index.html',zurl))
print u'正在存儲----%s' %ztitle
mysql.addchapters(lastrowid,ztitle,content) 3、執(zhí)行腳本
4、查看數(shù)據(jù)庫
可以看到已經(jīng)存儲成功了。
報錯：
_mysql_exceptions.OperationalError: (1364, "Field 'novelid' doesn't have a default value")
解決：執(zhí)行sql語句
SELECT @@GLOBAL.sql_mode;
SET @@GLOBAL.sql_mode="NO_ENGINE_SUBSTITUTION";
報錯參考：http://blog.sina.com.cn/s/blog_6d2b3e4901011j9w.html
