Twitter爬虫核心技术：全自动抓取世界上的主题以及转推,引用,回复的用户的微博

3/3Twitter爬虫核心技术：全自动抓取世界上的主题以及转推,引用,回复的用户的微博.importtwitter

defoauth_login():

#XXX:Gotohttp://./doc/7e279bdc48649b6648d7c1c708a1284ac85005ff.html/apps/newtocreateanappandgetvalues#forthesecredentialsthatyou'llneedtoprovideinplaceofthese

#emptystringvaluesthataredefinedasplaceholders.

#Seehttps://http://./doc/7e279bdc48649b6648d7c1c708a1284ac85005ff.html/docs/auth/oauthformoreinformation

#onTwitter'sOAuthimplementation.

CONSUMER_KEY=''

CONSUMER_SECRET=''

OAUTH_TOKEN=''

OAUTH_TOKEN_SECRET=''

auth=twitter.oauth.OAuth(OAUTH_TOKEN,OAUTH_TOKEN_SECRET,

CONSUMER_KEY,CONSUMER_SECRET)

twitter_api=twitter.Twitter(auth=auth)

returntwitter_api

#Sampleusage

twitter_api=oauth_login()

#Nothingtoseebydisplayingtwitter_apiexceptthatit'snowa

#definedvariable

printtwitter_api

importjson

deftwitter_trends(twitter_api,woe_id):

#PrefixIDwiththeunderscoreforquerystringparameterization.

#Withouttheunderscore,thetwitterpackageappendstheIDvalue

#totheURLitselfasaspecial-casekeywordargument.

returntwitter_api.trends.place(_id=woe_id)

deftwitter_search(twitter_api,q,max_results=2000000,**kw):

#Seehttps://http://./doc/7e279bdc48649b6648d7c1c708a1284ac85005ff.html/docs/api/1.1/get/search/tweetsand

#https://http://./doc/7e279bdc48649b6648d7c1c708a1284ac85005ff.html/docs/using-searchfordetailsonadvanced

#searchcriteriathatmaybeusefulforkeywordarguments

#Seehttps://http://./doc/7e279bdc48649b6648d7c1c708a1284ac85005ff.html/docs/api/1.1/get/search/tweets

search_results=twitter_api.search.tweets(q=q,count=200000,**kw)

statuses=search_results['statuses']

#Iteratethroughbatchesofresultsbyfollowingthecursoruntilwe

#reachthedesirednumberofresults,keepinginmindthatOAuthusers#can"only"make180searchqueriesper15-minuteinterval.See

#https://http://./doc/7e279bdc48649b6648d7c1c708a1284ac85005ff.html/docs/rate-limiting/1.1/limits

#fordetails.Areasonablenumberofresultsis~1000,although

#thatnumberofresultsmaynotexistforallqueries.

#Enforceareasonablelimit

max_results=min(1000,max_results)

for_inrange(10):#10*100=1000

try:

next_results=search_results['search_metadata']['next_results']exceptKeyError,e:#Nomoreresultswhennext_resultsdoesn'texistbreak

#Createadictionaryfromnext_results,whichhasthefollowingform:

#?max_id=313519052523986943&q=NCAA&include_entities=1

kwargs=dict([kv.split('=')

forkvinnext_results[1:].split("&")])

search_results=twitter_api.search.tweets(**kwargs)

statuses+=search_results['statuses']

iflen(statuses)>max_results:

break

returnstatuses

defextract_tweet_entities(statuses):

#Seehttps://http://./doc/7e279bdc48649b6648d7c1c708a1284ac85005ff.html/docs/tweet-entitiesformoredetailsontweet

#entities

iflen(statuses)==0:

return[],[],[],[],[]

screen_names=[user_mention['screen_name']

forstatusinstatuses

foruser_mentioninstatus['entities']['user_mentions']]

hashtags=[hashtag['text']

forstatusinstatuses

forhashtaginstatus['entities']['hashtags']]

urls=[url['expanded_url']

forstatusinstatuses

forurlinstatus['entities']['urls']]

symbols=[symbol['text']

forstatusinstatuses

forsymbolinstatus['entities']['symbols']]

#Insomecircumstances(suchassearchresults),themediaentity

#maynotappear

ifstatus['entities'].has_key('media'):

media=[media['url']

forstatusinstatuses

formediainstatus['entities']['media']]else:

media=[]

returnscreen_names,hashtags,urls,media,symbols

deffind_po