132 lines
4.4 KiB
Python
132 lines
4.4 KiB
Python
|
import requests
|
||
|
|
import facebook
|
||
|
|
import os
|
||
|
|
from elasticsearch import Elasticsearch
|
||
|
|
from datetime import datetime
|
||
|
|
from datetime import timedelta
|
||
|
|
|
||
|
|
app_id=os.environ['FB_APP_ID']
|
||
|
|
app_secret=os.environ['FB_APP_SECRET']
|
||
|
|
index_name=''
|
||
|
|
page_name=''
|
||
|
|
|
||
|
|
def get_fb_token(app_id, app_secret):
|
||
|
|
payload = {'grant_type': 'client_credentials', 'client_id': app_id, 'client_secret': app_secret}
|
||
|
|
file = requests.post('https://graph.facebook.com/oauth/access_token?', params = payload)
|
||
|
|
#print file.text #to test what the FB api responded with
|
||
|
|
result = file.text.split("=")[1]
|
||
|
|
#print file.text #to test the TOKEN
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
access_token=get_fb_token(app_id,app_secret)
|
||
|
|
week_delta = timedelta(days=7)*4
|
||
|
|
es = Elasticsearch()
|
||
|
|
es.indices.create(index=index_name, ignore=400)
|
||
|
|
graph = facebook.GraphAPI(access_token=access_token, version='2.7')
|
||
|
|
page_obj = graph.get_object(page_name)
|
||
|
|
|
||
|
|
def index_post(post,index_name):
|
||
|
|
es.index(
|
||
|
|
index=index_name,
|
||
|
|
doc_type="data",
|
||
|
|
id=post[u'id'],
|
||
|
|
timestamp=post[u'created_time'],
|
||
|
|
body=post)
|
||
|
|
def index_comments(post,index_name):
|
||
|
|
es.index(
|
||
|
|
index=index_name,
|
||
|
|
doc_type="comment",
|
||
|
|
id=post[u'id'],
|
||
|
|
timestamp=post[u'created_time'],
|
||
|
|
body=post)
|
||
|
|
def index_like(post,index_name):
|
||
|
|
es.index(
|
||
|
|
index=index_name,
|
||
|
|
doc_type="like",
|
||
|
|
id=post[u'like_id'],
|
||
|
|
timestamp=post[u'like_time'],
|
||
|
|
body=post)
|
||
|
|
|
||
|
|
def getfbpostsfrompage(fb_graph,page_id,field_list,time_since,time_until):
|
||
|
|
all_posts = []
|
||
|
|
res = fb_graph.get_object('/'+
|
||
|
|
page_id+
|
||
|
|
'/posts?fields='+','.join(field_list)+
|
||
|
|
'&since='+time_since+
|
||
|
|
'&until='+time_until)
|
||
|
|
while(True):
|
||
|
|
try:
|
||
|
|
for page in res[u'data']:
|
||
|
|
all_posts.append(page)
|
||
|
|
res=requests.get(res['paging']['next']).json()
|
||
|
|
except KeyError:
|
||
|
|
break
|
||
|
|
return all_posts
|
||
|
|
|
||
|
|
|
||
|
|
def getpostmetacomplet(fb_graph,post_id,field_list):
|
||
|
|
post_meta_complet = fb_graph.get_object('/'+
|
||
|
|
post_id+
|
||
|
|
'?fields='+','.join(field_list))
|
||
|
|
return post_meta_complet
|
||
|
|
|
||
|
|
def getpostreact(fb_graph,post_id,field_list,react_type,filter_type):
|
||
|
|
res = fb_graph.get_object('/'+post_id+
|
||
|
|
'/'+react_type+'/?fields='+','.join(field_list)+
|
||
|
|
'&filter='+filter_type)
|
||
|
|
all_comments = []
|
||
|
|
while(True):
|
||
|
|
try:
|
||
|
|
for comment in res[u'data']:
|
||
|
|
all_comments.append(comment)
|
||
|
|
res=requests.get(res[u'paging'][u'next']).json()
|
||
|
|
except KeyError:
|
||
|
|
break
|
||
|
|
return all_comments
|
||
|
|
|
||
|
|
def dict_update(l,x):
|
||
|
|
l.update(x)
|
||
|
|
return l
|
||
|
|
|
||
|
|
res = getfbpostsfrompage(graph,
|
||
|
|
page_obj[u'id'],
|
||
|
|
['id','created_time'],
|
||
|
|
(datetime.now().date()-week_delta).isoformat(),
|
||
|
|
datetime.now().date().isoformat())
|
||
|
|
|
||
|
|
for pp in res:
|
||
|
|
# Post
|
||
|
|
post_complet = getpostmetacomplet(graph,
|
||
|
|
pp[u'id'],
|
||
|
|
['message','created_time','id',
|
||
|
|
'status_type','shares','link',
|
||
|
|
'via'])
|
||
|
|
# Like
|
||
|
|
all_post_likes = getpostreact(graph,pp[u'id'],
|
||
|
|
['id','name'],
|
||
|
|
'likes',
|
||
|
|
'stream')
|
||
|
|
like_count = len(all_post_likes)
|
||
|
|
post_complet.update({u'like_count':like_count})
|
||
|
|
# Sauvegarde des "post"
|
||
|
|
index_post(post_complet,index_name)
|
||
|
|
# Sauvegarde des "like"
|
||
|
|
like_dicts = [dict_update(x,{u'like_time':pp['created_time'],
|
||
|
|
u'like_id':pp[u'id']+'_'+x['id']}) for x in all_post_likes]
|
||
|
|
for l in like_dicts:
|
||
|
|
index_like(l,index_name)
|
||
|
|
# Comments
|
||
|
|
res_comments = getpostreact(graph,pp[u'id'],['id'],'comments','stream')
|
||
|
|
for cc in res_comments:
|
||
|
|
comment_complet = getpostmetacomplet(graph,
|
||
|
|
cc[u'id'],
|
||
|
|
['id','from','message',
|
||
|
|
'created_time','comment_count','like_count',
|
||
|
|
'parent'])
|
||
|
|
# Sauvegarde des "comments"
|
||
|
|
index_comments(comment_complet,index_name)
|
||
|
|
print pp
|
||
|
|
|
||
|
|
|