抓取微博示例程序

王珂

由于在做舆情处理，需要大量的数据，所以就出现了这个东西，调用的微博公共API，如果有问题
查询微博文档
代码如下：

1. #encoding = utf8
   
2. 
   
3. import weibo
   
4. import sys, os, urllib, urllib2
   
5. import json
   
6. import cookielib
   
7. import ConfigParser
   
8. import time
   
9. import MySQLdb
   
10. import re
    
11. 
    
12. reload(sys)
    
13. sys.setdefaultencoding('utf8')
    
14. 
    
15. db_host=""
    
16. db_data=""
    
17. db_user=""
    
18. db_pass=""
    
19. db_port=""
    
20. 
    
21. APP_KEY="162454*****" #你的APP_KEY
    
22. APP_SECRET="753847a8d16ead8a8e7618*****************"#你的app_secret
    
23. REDIRECT_URL="https://api.weibo.com/oauth2/default.html"
    
24. 
    
25. USER_ID="crawlertest@sina.cn"
    
26. USER_PASS="12345*******"
    
27. client=weibo.APIClient(APP_KEY,APP_SECRET,REDIRECT_URL)
    
28. def make_access_token():
    
29.     params = urllib.urlencode({
    
30.     'action':'submit',
    
31.     'withOfficalFlag':'0',
    
32.     'ticket':'',
    
33.     'isLoginSina':'',
    
34.     'response_type':'code',
    
35.     'regCallback':'',
    
36.     'redirect_uri':REDIRECT_URL,
    
37.     'client_id':APP_KEY,
    
38.     'state':'',
    
39.     'from':'',
    
40.     'userId':USER_ID,
    
41.     'passwd':USER_PASS,
    
42.     })
    
43.     login_url = 'https://api.weibo.com/oauth2/authorize'
    
44.     url = client.get_authorize_url()
    
45.     content = urllib2.urlopen(url)
    
46.     if content:
    
47.         headers = { 'Referer' : url }
    
48.     request = urllib2.Request(login_url, params, headers)
    
49.     opener = get_opener(False)
    
50.     urllib2.install_opener(opener)
    
51.     try:
    
52.         f = opener.open(request)
    
53.         return_redirect_uri = f.url
    
54.     except urllib2.HTTPError, e:
    
55.         return_redirect_uri = e.geturl()
    
56.    
    
57.     code = return_redirect_uri.split('=')[1]
    
58.    
    
59.     token = client.request_access_token(code,REDIRECT_URL)
    
60.     save_access_token(token)
    
61.    
    
62. def get_opener(proxy=False):
    
63.     rv=urllib2.build_opener(get_cookie(), SmartRedirectHandler())
    
64.     rv.addheaders = [('User-agent', 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)')]
    
65.     return rv
    
66. 
    
67. def get_cookie():
    
68.     cookies = cookielib.CookieJar()
    
69.     return urllib2.HTTPCookieProcessor(cookies)
    
70. 
    
71. class SmartRedirectHandler(urllib2.HTTPRedirectHandler):
    
72.     def http_error_301(self,cls, req, fp, code, msg, headers):
    
73.         result = urllib2.HTTPRedirectHandler.http_error_301(cls, req, fp, code, msg, headers)
    
74.         result.status = code
    
75.         print headers
    
76.         return result
    
77. 
    
78.     def http_error_302(self,cls, req, fp, code, msg, headers):
    
79.         result = urllib2.HTTPRedirectHandler.http_error_302(cls, req, fp, code, msg, headers)
    
80.         result.status = code
    
81.         print headers
    
82.         return result
    
83.    
    
84. def save_access_token(token):
    
85.     time=time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
    
86.     cf=ConfigParser.ConfigParser()
    
87.     cf.read("./token.ini")
    
88.     cf.set("token","token",token.access_token)
    
89.     cf.set("token","expires_in",token.expires_in )
    
90.     cf.set("token","data",time)   
    
91.     cf.write(open("./token.ini","w"))
    
92.    
    
93. def apply_access_token():
    
94.     cf=ConfigParser.ConfigParser()
    
95.     cf.read("./token.ini")
    
96.     access_token=cf.get("token","access_token")
    
97.     expires_in=cf.get("token","expires_in")
    
98.     #print access_token
    
99.     #print expires_in
    
100.    
     
101.     client.set_access_token(access_token, expires_in)
     
102.    
     
103. def get_public_weibo():
     
104.     try:
     
105.         text=client.statuses.public_timeline.get()
     
106.     except StandardError,e:
     
107.         if e.error_code==10023:
     
108.             print u"当前账号已经超出使用限制，请稍候再试！"
     
109.             exit(0)
     
110.         time.sleep(3600)
     
111.     js=json.dumps(text)
     
112.     js=json.loads(js)
     
113.     content=js['statuses']
     
114.     lenth=len(content)
     
115.     max=js['total_number']
     
116.     for i in range(0,lenth-1):
     
117.         try:
     
118.             source=content[i]["source"]
     
119.             source=re.match(r'''<a href="(.+?)" rel="nofollow">(.+?)</a>''',source)
     
120.             date=content[i]['created_at']
     
121.             mid=content[i]["idstr"]
     
122.             text=content[i]['text']
     
123.             source=source.group(2)
     
124.             reports=content[i]['reposts_count']
     
125.             comments=content[i]['comments_count']
     
126.             atti=content[i]['attitudes_count']
     
127.             uid=content[i]['user']['id']
     
128.             nick=content[i]['user']['screen_name']
     
129.             loca=content[i]['user']['location']
     
130.             deci=content[i]['user']['description']
     
131.             gender=content[i]['user']['gender']
     
132.             verti=content[i]['user']['verified']
     
133.             follws=content[i]['user']['bi_followers_count']
     
134.             value=[date.decode(),mid,text,source,reports,comments,atti,uid,nick,loca,deci,gender,verti,follws]
     
135.             db_save_info(value)
     
136.             print u"插入第%s条数据成功！"%i
     
137.         except:
     
138.             pass
     
139.    
     
140. 
     
141.         
     
142.         
     
143.         
     
144.         
     
145.         
     
146. def db_get_config():
     
147.     cf=ConfigParser.ConfigParser()
     
148.     cf.read("./token.ini")
     
149.     db_host=cf.get("dbinfo", "dbserver")
     
150.     db_data=cf.get("dbinfo","db_data")
     
151.     db_user=cf.get("dbinfo","db_user")
     
152.     db_pass=cf.get("dbinfo","password")
     
153.     db_port=cf.get("dbinfo","port")
     
154.    
     
155. 
     
156. def db_save_info(value):
     
157.     #value=['2012','10000000','test','test','2','2','2','32111','fuck','1','1','1','1']
     
158.     conn=MySQLdb.connect("127.0.0.1","root","123456",charset="utf8")
     
159.     conn.select_db("db_info")
     
160.     cursor=conn.cursor()
     
161.     cursor.execute("SET NAMES 'utf8'")
     
162.     cursor.execute('''insert into weibo_info values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)''',value)
     
163.     conn.commit()
     
164.     cursor.close()
     
165. 
     
166.    
     
167. def get_db_count():
     
168.     conn=MySQLdb.connect("127.0.0.1","root","123456")
     
169.     conn.select_db("db_info")
     
170.     cursor=conn.cursor()  
     
171.     cursor.execute("select count(*) from weibo_info")
     
172.     result=cursor.fetchall()
     
173.     for i in result:
     
174.         result=i[0]
     
175.     return result
     
176. 
     
177. 
     
178. if __name__=="__main__":
     
179.     print u'''
     
180. ###################################  
     
181. # [url=home.php?mod=space&uid=11566]@Author[/url] iceforce  
     
182. # @date 2014-11-17  
     
183. ##################################  
     
184. #抓取公共微博信息示例
     
185. #  
     
186. ##################################
     
187.             '''
     
188.     count=get_db_count()
     
189.     apply_access_token()
     
190.     for i in range(0,150):
     
191.         count1=get_db_count()
     
192.         count2=count1-count        
     
193.         print u"=======================================第%s次执行,已插入%s条数据============================================="%(i,count2)
     
194.         get_public_weibo()
     
195. 
     
196.     print u"============================================执行完成！============================================================"
     
197. 
     
198.    
     

复制代码

Jack-5

学习学习技术，加油！

wtsqq123

学习学习技术，加油！

arctic

还是不错的哦，顶了

H.U.C-麦麦

感谢楼主的分享~

r00tc4

支持中国红客联盟(ihonker.org)

热心网友2

感谢分享这个微博数据抓取的示例程序，对做舆情分析很有参考价值。代码结构清晰，包含了OAuth授权流程、cookie和重定向处理，以及access_token的本地保存，正好可以学习一下怎么通过公开API获取微博数据。准备有空也试试。