archive / 2009–2013
Facebook Data Harvester
[exec]
include('face-harvie.inc.php'); //include our custom functions
$time = microtime();
$time = explode(' ', $time);
$time = $time[1] + $time[0];
$start = $time;
mysql_connect("192.168.131.126", "papahet", "n0t0f0x2") or die(mysql_error());
mysql_select_db("FBtest") or die(mysql_error());
// Facebook App Config Settings
$app_id = '278566165589449';
$app_secret = '495f9a9821f9157ee4c7d5a5509dc82e';
$my_url = 'http://ryrobes.com/getfacedata/'; //change depending on shit.
$permission_scope = 'read_insights,read_stream,friends_location,friends_hometown,user_location,user_photos,user_status,friends_status,friends_birthday,user_birthday,user_relationships,friends_relationships,user_about_me,friends_about_me,user_interests,friends_interests,user_religion_politics,friends_religion_politics,user_activities,friends_activities,friends_subscriptions,user_likes,friends_likes,user_checkins,friends_checkins,email,manage_pages';
if(!$_REQUEST['pre_access_token']) {
session_start();
$code = $_REQUEST["code"];
if(empty($code)) {
$_SESSION['state'] = md5(uniqid(rand(), TRUE)); //CSRF protection
$dialog_url = "http://www.facebook.com/dialog/oauth?client_id=" . $app_id . "&redirect_uri=" . urlencode($my_url) . "&state=" . $_SESSION['state'] . "&scope=" . $permission_scope . "&display=page";
echo("");
}
///issues.
////if($_SESSION['state'] && ($_SESSION['state'] === $_REQUEST['state'])) {
if($_SESSION['state']) { ///test?
$token_url = "https://graph.facebook.com/oauth/access_token?". "client_id=" . $app_id . "&redirect_uri=" . urlencode($my_url) . "&client_secret=" . $app_secret . "&code=" . $code . "&display=page";
$response = file_get_contents($token_url);
$params = null;
parse_str($response, $params);
$graph_url = "https://graph.facebook.com/me?access_token=" . $params['access_token'];
$access_token = 'access_token='.$params['access_token'];
$expires = $params['expires'];
//echo 'token expires: '.$expires; //debug - pass to nohup command line as param 2?
$user = json_decode(file_get_contents($graph_url)); //simple connection test using Open Graph API (no FQL)
//echo("Hello " . $user->name . ' ' . $user->id . '
');
$pre_access_token = $params['access_token']; //for compat
//try to figure out what to do with the user (new user, old (still good) user, old (expired) user - plus how old their data is)
$check_status = "select 1 from FBtest.RunDetails where access_token = '".$access_token."' and uid = '".$user->id."' limit 0,1 ";
$result = mysql_query($check_status);
$row = mysql_fetch_row($result);
if ($row[0] == '1') {
//echo '
Known User; still on good (last) token.
'; } //echo $check_status.'';; } else { //fail auth hard. to prevent CSRF scammers echo("
Oh noes. The session state does not match. Cannot validate!
[button size='xl' link='http://ryrobes.com/getfacedata/']Try again, please?[/button]
");
//session_destroy();
break;
}
} else { //also should check is pre-access-token is good, but whatever - its for debugging
//echo 'pre auth token passed: '.$_REQUEST['pre_access_token'];
$access_token = 'access_token='.$_REQUEST['pre_access_token'];
//echo 'test';
}
exec("kill -9 ps -ef | grep ".$pre\_access\_token." | grep -v grep | awk '{print $2}'");
//echo $access_token . '
';
//echo 'You have been authed!!
';
$fql = "SELECT name, uid, pic_square, username, proxied_email, email FROM user WHERE uid = me()";
$fql_query_obj = ExecuteFQLquery($fql, $access_token, '999999');
foreach($fql_query_obj['data'] as $k=>$v) {
$req_user_uid = sprintf("%.0f", $v['uid']);
$name = $v['name'];
$username = $v['username'];
$email = $v['email'];
$proxied_email = $v['proxied_email'];
$pic_square = $v['pic_square']; }
///echo '
'.$name.'
'; echo '[box type="info" icon="'.$pic_square.'"]
'.$name.'
[/box]Note: I should probably add an option to email the result link if peeps dont want to watch this...'; ////echo '[button color="grey"]Fucking buttons![/button]'; ///insert into StateCheck - $access_token $_SESSION['state'] $_REQUEST['state'] //if($access_token && $_SESSION['state'] && $_REQUEST['state']) { mysql_query("insert into FBtest.StateCheck (access_token, session_state, request_state, remote_ip, browser) values ('".$access_token."', '".$_SESSION['state']."', '".$_REQUEST['state']."', '".$_SERVER['REMOTE_ADDR']."', '".$_SERVER['HTTP_USER_AGENT']."' )"); //} if(!$_REQUEST['pickup'] == 'gimmie') { //assuming that they aren't back to get stuff, let's set off the trunc-n-load (again) exec("nohup php /srv/www/ryrobes.com/public_html/face-harvie.php ".str_replace('access_token=','',$access_token)." ".$expires." >/tmp/".$access_token." 2>/tmp/".$access_token." &"); sleep(2); //in order to let the counter reset BEFORE it appears (ajaxy) echo ' '; echo 'We are now pulling your raw Facebook data via FB'."'".'s API (using FQL queries) - it may take quite awhile (especially if you have many fan pages that you are an admin of). [typography color="black"]Please leave this page open, a link will appear when it has finished to take you to the "data download" page.[/typography]
Disclaimer: None of this data is kept, it gets erased a few hours after it has been downloaded (just in case you were wondering).
Check out my blog and some fun DataViz projects while you wait!
[sixcol_one]
[/sixcol_one][sixcol_one]
[/sixcol_one][sixcol_one]
[/sixcol_one][sixcol_one]
[/sixcol_one][sixcol_one]
[/sixcol_one][sixcol_one_last]
[/sixcol_one_last]
Issues? Please report in the comments below - consider this v 0.001, I'."'".'m sure it'."'".'s quite buggy and I haven'."'".'t had a chance to test with that many different profiles. So any constructive feedback is welcome!
';
} else {
//echo 'YES!';
$dbhost = "192.168.131.126";
$dbuser = "papahet";
$dbpwd = "n0t0f0x2";
$dbname = "fb_user_".$req_user_uid."";
$dumpfile = $dbname . "_" . date("Y-m-d_H-i-s") . ".sql";
exec("/usr/bin/mysqldump --opt --host=$dbhost --user=$dbuser --password=$dbpwd $dbname > /srv/www/ryrobes.com/public_html/facedata/$dumpfile");
$filesizebytes_un = human_filesize(filesize("/srv/www/ryrobes.com/public_html/facedata/$dumpfile"));
exec("gzip /srv/www/ryrobes.com/public_html/facedata/$dumpfile");
$filesizebytes_cmp = human_filesize(filesize("/srv/www/ryrobes.com/public_html/facedata/$dumpfile.gz"));
//require_once("phpchart/conf.php");
//$pc = new C_PhpChartX(array(array(11, 9, 5, 12, 14)),'basic_chart');
//$pc->draw();
$uid = $req_user_uid;
echo '[box]';
$result = mysql_query("SELECT pic_square from fb_user_".$uid.".Friends");
while($row = mysql_fetch_array($result))
{
$fpic = $row[0];
echo '';
}
echo '[/box]';
$result = mysql_query("SELECT (select count(*) from fb_user_".$uid.".Friends),
(select count(*) from fb_user_".$uid.".Checkin),
(select count(*) from fb_user_".$uid.".Comments),
(select count(*) from fb_user_".$uid.".Likes),
(select count(*) from fb_user_".$uid.".Stream),
(select count(*) from fb_user_".$uid.".PageFans),
(select count(*) from fb_user_".$uid.".Pages),
(select count(*) from fb_user_".$uid.".PageAdmin),
(select count(*) from fb_user_".$uid.".Family),
(select count(*) from fb_user_".$uid.".Insights)");
while($row = mysql_fetch_array($result))
{
$friends = $row[0];
$checkins = $row[1];
$comments = $row[2];
$likes = $row[3];
$stream = $row[4];
$pagefans = $row[5];
$pages = $row[6];
$adminpages = $row[7];
$family = $row[8];
$insights = $row[9];
}
echo '
[easychart type="horizbar" height="40" title="Breakdown of '.$name."'".'s FQL Facebook Tables Pulled" groupnames="Records" groupcolors="6d6d6f" valuenames="All Friend / Your Check-ins,All Comments,Your Friends,All Family Connections,Page Insights,All Likes,Fan Page Connections,Fan Page Details,Your Entire Stream" group1values="'.$checkins.','.$comments.','.$friends.','.$family.','.$insights.','.$likes.','.$pagefans.','.$pages.','.$stream.'" ]';
echo '
Download MySQL database in SQL format:
'; echo 'This is a SQL extract of all tables below in MySQL format - created with MySQLDump.'; echo '[box type="download"]'.$dumpfile.'.gz '.$filesizebytes_cmp.' ('.$filesizebytes_un.' uncompressed)[/box][hr]'; echo '
Very simple example of the kind of things you can learn with your raw insights data.
But this is only the tip of the iceberg - remember you also have all your post, like and comment data as well...
[hr]'; echo '
Download each table as CSV files:
';
echo '[box type="download"]Checkin Table
This table contains all checkins available - including yours and your friends, complete with Latitude and Longitude.[/box]';
echo '[box type="download"]Comments Table
This table contains all comments on your posts (status, check-ins, uploads, etc), for both your personal page as well as any pages that you are an admin of.[/box]';
echo '[box type="download"]Friends Table
This table contains all available data on your friends.[/box]';
echo '[box type="download"]Insights Table
This table contains your "daily" PageFan insights from January 2010 to data for all Pages that you are an admin of.[/box]';
echo '[box type="download"]Likes Table
This table contains all likes on your posts (status, check-ins, uploads, etc), for both your personal page as well as any pages that you are an admin of.[/box]';
echo '[box type="download"]PageAdmin Table
This table contains any pages that you are an admin of.[/box]';
echo '[box type="download"]PageFans Table
This table contains all pages liked by all your (personal) friends (pages that they are "fans" of).[/box]';
echo '[box type="download"]Pages Table
This table contains all page details for all pages that you or your friends have "liked".[/box]';
echo '[box type="download"]Stream Table
This table contains all content created by you or your associated pages. Basically any action you have ever done on Facebook (liking, commenting, posting, etc.)[/box]';
echo '
I obviously didn'."'".'t map out all the links, but this will give you a head start. For the full documentation, check out the Facebook Developer FQL table & field descriptions, since I pretty much pulled each table as they exist in FQL. As always, please let me know all issues below - I'."'".'m SURE that this has many bugs and I didn'."'".'t get a chance to test it on a large number of profiles! '; } [/exec]