balkian/bitter
1
0
Fork 0
mirror of https://github.com/balkian/bitter.git synced 2024-12-22 00:18:12 +00:00
Code Issues Releases Wiki Activity

Added webserver

Browse Source
This commit is contained in:
J. Fernando Sánchez 2016-09-14 19:53:56 +02:00
parent 97028e38b1
commit 17f589c710
17 changed files with 612 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
README.md Normal file
View File

@ -0,0 +1,50 @@
#Description
There are two parts to bitter.
First of all, it is a wrapper over Python twitter that adds support for several Twitter API credentials (e.g. authorizing the same app with different user accounts).
Secondly, it is a command line tool to automate several actions (e.g. downloading user networks) using the wrapper.
# Instructions
In the command line:
python -m bitter --help
or
bitter --help
Programmatically:
```python
from bitter.crawlers import TwitterQueue
wq = TwitterQueue.from_credentials()
print(wq.users.show(user_name='balkian'))
```
# Credentials format
```
{"user": "balkian", "consumer_secret": "xxx", "consumer_key": "xxx", "token_key": "xxx", "token_secret": "xxx"}
```
By default, bitter uses '~/.bitter-credentials.json', but you may choose a different file:
```
python -m bitter -c <credentials_file> ...
```
# Server
To add more users to the credentials file, you may run the builtin server, with the consumer key and secret of your app:
```
python -m bitter server <consumer_key> <consumer_secret>
```
# Notice
Please, use according to Twitter's Terms of Service
# TODO
* Tests
* Docs

25
bitter/cli.py
View File

@ -20,21 +20,15 @@ logger = logging.getLogger(__name__)
@click.option("--verbose", is_flag=True)
@click.option("--logging_level", required=False, default='WARN')
@click.option("--config", required=False)
@click.option('-c', '--credentials',show_default=True, default='credentials.json')
@click.option('-c', '--credentials', show_default=True, default='~/.bitter-credentials.json')
@click.pass_context
def main(ctx, verbose, logging_level, config, credentials):
logging.basicConfig(level=getattr(logging, logging_level))
ctx.obj = {}
ctx.obj['VERBOSE'] = verbose
ctx.obj['CONFIG'] = config
if os.path.isfile(credentials):
ctx.obj['CREDENTIALS'] = credentials
else:
global_file = os.path.expanduser('~/.bitter-credentials.json')
if os.path.isfile(global_file):
ctx.obj['CREDENTIALS'] = global_file
else:
raise Exception('You need to provide a valid credentials file')
utils.create_credentials(credentials)
@main.group()
@click.pass_context
@ -46,8 +40,7 @@ def tweet(ctx):
@click.pass_context
def get_tweet(ctx, tweetid):
wq = crawlers.TwitterQueue.from_credentials(ctx.obj['CREDENTIALS'])
c = wq.next()
t = crawlers.get_tweet(c.client, tweetid)
t = utils.get_tweet(wq, tweetid)
print(json.dumps(t, indent=2))
@ -320,5 +313,17 @@ def get_limits(ctx, url):
else:
print(json.dumps(resp, indent=2))
@main.command('server')
@click.argument('CONSUMER_KEY', required=True)
@click.argument('CONSUMER_SECRET', required=True)
@click.pass_context
def run_server(ctx, consumer_key, consumer_secret):
from . import config
config.CONSUMER_KEY = consumer_key
config.CONSUMER_SECRET = consumer_secret
from .webserver import app
app.run()
if __name__ == '__main__':
main()

13
bitter/config.py Normal file
View File

@ -0,0 +1,13 @@
'''
Common configuration for other modules.
It is not elegant, but it works with flask and the oauth decorators.
Using this module allows you to change the config before loading any other module.
E.g.:
import bitter.config as c
c.CREDENTIALS="/tmp/credentials"
from bitter.webserver import app
app.run()
'''
CREDENTIALS = '~/.bitter-credentials.json'

8
bitter/crawlers.py
View File

@ -8,6 +8,8 @@ logger = logging.getLogger(__name__)
from twitter import *
from collections import OrderedDict
from . import utils
from . import config
class AttrToFunc(object):
@ -102,12 +104,10 @@ class TwitterQueue(AttrToFunc):
return self.next().client
@classmethod
def from_credentials(self, cred_file):
def from_credentials(self, cred_file=None):
wq = TwitterQueue()
with open(cred_file) as f:
for line in f:
cred = json.loads(line)
for cred in utils.get_credentials(cred_file):
c = Twitter(auth=OAuth(cred['token_key'],
cred['token_secret'],
cred['consumer_key'],

50
bitter/static/css/style.css Normal file
View File

@ -0,0 +1,50 @@
@charset 'UTF-8';
html {
display: block;
margin: auto;
width: 100%;
background-color: black;
color: white;
font-size: 1.5em;
}
.wrapper {
margin: 0 auto;
width: 80%;
text-align: center;
}
.me {
width: 15em;
border-radius: 150px;
-webkit-border-radius: 150px;
-moz-border-radius: 150px;
}
.button {
background-color: black;
color: white;
padding: 0.5em;
font-size: 1.5em;
border: 3px solid white;
margin-top: 1em;
display: inline-block;
border-radius: 1em;
text-decoration: none;
font-weight: bold;
}
.button:hover {
color: black;
background-color: white;
}
.limits {
text-align: left;
}
.jsonlimits {
overflow: auto;
max-height: 5em;
}

BIN
bitter/static/images/me.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 104 KiB

241
bitter/static/index.html Normal file
View File

@ -0,0 +1,241 @@
<!DOCTYPE HTML>
<!--
Miniport 2.5 by HTML5 UP
html5up.net | @n33co
Free for personal and commercial use under the CCA 3.0 license (html5up.net/license)
-->
<html>
<head>
<title>Miniport by HTML5 UP</title>
<meta http-equiv="content-type" content="text/html; charset=utf-8" />
<meta name="description" content="" />
<meta name="keywords" content="" />
<link href="http://fonts.googleapis.com/css?family=Open+Sans:300,600,700" rel="stylesheet" />
<script src="js/jquery.min.js"></script>
<script src="js/config.js"></script>
<script src="js/skel.min.js"></script>
<noscript>
<link rel="stylesheet" href="css/skel-noscript.css" />
<link rel="stylesheet" href="css/style.css" />
<link rel="stylesheet" href="css/style-desktop.css" />
</noscript>
<!--[if lte IE 9]><link rel="stylesheet" href="css/ie9.css" /><![endif]-->
<!--[if lte IE 8]><script src="js/html5shiv.js"></script><link rel="stylesheet" href="css/ie8.css" /><![endif]-->
<!--[if lte IE 7]><link rel="stylesheet" href="css/ie7.css" /><![endif]-->
</head>
<body>
<!-- Nav -->
<nav id="nav">
<ul class="container">
<li><a href="#top">Top</a></li>
<li><a href="#work">Work</a></li>
<li><a href="#portfolio">Portfolio</a></li>
<li><a href="#contact">Contact</a></li>
</ul>
</nav>
<!-- Home -->
<div class="wrapper wrapper-style1 wrapper-first">
<article class="container" id="top">
<div class="row">
<div class="4u">
<span class="me image image-full"><img src="images/me.jpg" alt="" /></span>
</div>
<div class="8u">
<header>
<h1>Hi. I'm <strong>Jane Doe</strong>.</h1>
</header>
<p>And this is <strong>Miniport</strong>, a free, fully responsive HTML5 site template designed by <a href="http://n33.co/">AJ</a> for <a href="http://html5up.net/">HTML5 UP</a> &amp; released under the <a href="http://html5up.net/license/">CCA license</a>.</p>
<a href="#work" class="button button-big">Learn about what I do</a>
</div>
</div>
</article>
</div>
<!-- Work -->
<div class="wrapper wrapper-style2">
<article id="work">
<header>
<h2>I design and build amazing things.</h2>
<span>Odio turpis amet sed consequat eget posuere consequat.</span>
</header>
<div class="container">
<div class="row">
<div class="4u">
<section class="box box-style1">
<span class="fa featured fa-comments-o"></span>
<h3>Consequat lorem</h3>
<p>Ornare nulla proin odio consequat sapien vestibulum ipsum primis sed amet consequat lorem dolore.</p>
</section>
</div>
<div class="4u">
<section class="box box-style1">
<span class="fa featured fa-file-o"></span>
<h3>Lorem dolor tempus</h3>
<p>Ornare nulla proin odio consequat sapien vestibulum ipsum primis sed amet consequat lorem dolore.</p>
</section>
</div>
<div class="4u">
<section class="box box-style1">
<span class="fa featured fa-thumbs-o-up"></span>
<h3>Feugiat posuere</h3>
<p>Ornare nulla proin odio consequat sapien vestibulum ipsum primis sed amet consequat lorem dolore.</p>
</section>
</div>
</div>
</div>
<footer>
<p>Lorem ipsum dolor sit sapien vestibulum ipsum primis?</p>
<a href="#portfolio" class="button button-big">See some of my recent work</a>
</footer>
</article>
</div>
<!-- Portfolio -->
<div class="wrapper wrapper-style3">
<article id="portfolio">
<header>
<h2>Awesome work makes happy clients.</h2>
<span>Proin odio consequat sapien vestibulum ipsum primis sed amet consequat lorem dolore feugiat lorem ipsum dolore.</span>
</header>
<div class="container">
<div class="row">
<div class="12u">
</div>
</div>
<div class="row">
<div class="4u">
<article class="box box-style2">
<a href="http://flypixel.com/generic-smartphone/8949517882265310" class="image image-full"><img src="images/portfolio01.jpg" alt="" /></a>
<h3><a href="http://flypixel.com/generic-smartphone/8949517882265310">Magna feugiat</a></h3>
<p>Ornare nulla proin odio consequat.</p>
</article>
</div>
<div class="4u">
<article class="box box-style2">
<a href="http://flypixel.com/n33" class="image image-full"><img src="images/portfolio02.jpg" alt="" /></a>
<h3><a href="http://flypixel.com/n33">Veroeros primis</a></h3>
<p>Ornare nulla proin odio consequat.</p>
</article>
</div>
<div class="4u">
<article class="box box-style2">
<a href="http://flypixel.com/wood-ui-kit/3574765984616310" class="image image-full"><img src="images/portfolio03.jpg" alt="" /></a>
<h3><a href="http://flypixel.com/wood-ui-kit/3574765984616310">Lorem ipsum</a></h3>
<p>Ornare nulla proin odio consequat.</p>
</article>
</div>
</div>
<div class="row">
<div class="4u">
<article class="box box-style2">
<a href="http://flypixel.com/n33-pattern-set-1/3522389001865317" class="image image-full"><img src="images/portfolio04.jpg" alt="" /></a>
<h3><a href="http://flypixel.com/n33-pattern-set-1/3522389001865317">Tempus dolore</a></h3>
<p>Ornare nulla proin odio consequat.</p>
</article>
</div>
<div class="4u">
<article class="box box-style2">
<a href="http://flypixel.com/cityscape/9803996277226316" class="image image-full"><img src="images/portfolio05.jpg" alt="" /></a>
<h3><a href="http://flypixel.com/cityscape/9803996277226316">Feugiat aliquam</a></h3>
<p>Ornare nulla proin odio consequat.</p>
</article>
</div>
<div class="4u">
<article class="box box-style2">
<a href="http://flypixel.com/n33" class="image image-full"><img src="images/portfolio06.jpg" alt="" /></a>
<h3><a href="http://flypixel.com/n33">Sed amet ornare</a></h3>
<p>Ornare nulla proin odio consequat.</p>
</article>
</div>
</div>
</div>
<footer>
<p>Lorem ipsum dolor sit sapien vestibulum ipsum primis?</p>
<a href="#contact" class="button button-big">Get in touch with me</a>
</footer>
</article>
</div>
<!-- Contact -->
<div class="wrapper wrapper-style4">
<article id="contact" class="container small">
<header>
<h2>Want to hire me? Get in touch!</h2>
<span>Ornare nulla proin odio consequat sapien vestibulum ipsum sed lorem.</span>
</header>
<div>
<div class="row">
<div class="12u">
<form method="post" action="#">
<div>
<div class="row half">
<div class="6u">
<input type="text" name="name" id="name" placeholder="Name" />
</div>
<div class="6u">
<input type="text" name="email" id="email" placeholder="Email" />
</div>
</div>
<div class="row half">
<div class="12u">
<input type="text" name="subject" id="subject" placeholder="Subject" />
</div>
</div>
<div class="row half">
<div class="12u">
<textarea name="message" id="message" placeholder="Message"></textarea>
</div>
</div>
<div class="row">
<div class="12u">
<a href="#" class="button form-button-submit">Send Message</a>
<a href="#" class="button button-alt form-button-reset">Clear Form</a>
</div>
</div>
</div>
</form>
</div>
</div>
<div class="row">
<div class="12u">
<hr />
<h3>Find me on ...</h3>
<ul class="social">
<li class="twitter"><a href="http://twitter.com/n33co" class="fa fa-twitter"><span>Twitter</span></a></li>
<li class="facebook"><a href="#" class="fa fa-facebook"><span>Facebook</span></a></li>
<li class="dribbble"><a href="http://dribbble.com/n33" class="fa fa-dribbble"><span>Dribbble</span></a></li>
<li class="linkedin"><a href="#" class="fa fa-linkedin"><span>LinkedIn</span></a></li>
<li class="tumblr"><a href="#" class="fa fa-tumblr"><span>Tumblr</span></a></li>
<li class="googleplus"><a href="#" class="fa fa-google-plus"><span>Google+</span></a></li>
<li class="github"><a href="http://github.com/n33" class="fa fa-github"><span>Github</span></a></li>
<!--
<li class="rss"><a href="#" class="fa fa-rss"><span>RSS</span></a></li>
<li class="instagram"><a href="#" class="fa fa-instagram"><span>Instagram</span></a></li>
<li class="foursquare"><a href="#" class="fa fa-foursquare"><span>Foursquare</span></a></li>
<li class="skype"><a href="#" class="fa fa-skype"><span>Skype</span></a></li>
<li class="soundcloud"><a href="#" class="fa fa-soundcloud"><span>Soundcloud</span></a></li>
<li class="youtube"><a href="#" class="fa fa-youtube"><span>YouTube</span></a></li>
<li class="blogger"><a href="#" class="fa fa-blogger"><span>Blogger</span></a></li>
<li class="flickr"><a href="#" class="fa fa-flickr"><span>Flickr</span></a></li>
<li class="vimeo"><a href="#" class="fa fa-vimeo"><span>Vimeo</span></a></li>
-->
</ul>
<hr />
</div>
</div>
</div>
<footer>
<ul id="copyright">
<li>&copy; 2013 Jane Doe</li>
<li>Images: <a href="http://fotogrph.com">fotogrph</a></li>
<li>Design: <a href="http://html5up.net/">HTML5 UP</a></li>
</ul>
</footer>
</article>
</div>
</body>
</html>

36
bitter/templates/base.html Normal file
View File

@ -0,0 +1,36 @@
<!DOCTYPE HTML>
<!--
Miniport 2.5 by HTML5 UP
html5up.net | @n33co
Free for personal and commercial use under the CCA 3.0 license (html5up.net/license)
-->
<html>
<head>
<title>Bitter's Login</title>
<meta http-equiv="content-type" content="text/html; charset=utf-8" />
<meta name="description" content="" />
<meta name="keywords" content="" />
<link rel="stylesheet" href="static/css/style.css" />
<!--[if lte IE 9]><link rel="stylesheet" href="static/css/ie9.css" /><![endif]-->
<!--[if lte IE 8]><script src="static/js/html5shiv.js"></script><link rel="stylesheet" href="static/css/ie8.css" /><![endif]-->
<!--[if lte IE 7]><link rel="stylesheet" href="static/css/ie7.css" /><![endif]-->
</head>
<body>
<!-- Nav -->
<!-- Home -->
<div class="wrapper">
<img class="me" src="static/images/me.jpg" alt="" />
<header>
{% block header %}
{% endblock %}
</header>
<div class="content">
{% block content %}
{% endblock %}
</div>
</div>
</body>
</html>

10
bitter/templates/home.html Normal file
View File

@ -0,0 +1,10 @@
{% extends "base.html" %}
{% block header %}
<h1>Hi. I'm a <strong>researcher</strong>.</h1>
{% endblock %}
{% block content %}
<p>Getting data from Twitter is hard, due to the limits imposed by the Twitter API</p>
<p>By logging in with my app you help me get more data for my research.</p>
<p>I will not use your personal information in any way.</p>
<a href="login" class="button button-big">Log in!</a>
{% endblock %}

11
bitter/templates/limits.html Normal file
View File

@ -0,0 +1,11 @@
{% extends "base.html" %}
{% block content %}
{% if limits | length %}
<ul id="double">
{% for limit in limits %}
<li class="limits"> <span>{{ limit }} </span><span><pre class="jsonlimits">{{ limits[limit] }}</pre></span></li>
{% endfor %}
</ul>
{% endif %}
{% endblock %}

8
bitter/templates/sad.html Normal file
View File

@ -0,0 +1,8 @@
{% extends "base.html" %}
{% block header %}
<h1>You don't wanna authorize me? :(</h1>
{% endblock %}
{% block content %}
<p>It's ok, but come back if you change your mind...</p>
{% endblock %}

10
bitter/templates/thanks.html Normal file
View File

@ -0,0 +1,10 @@
{% extends "base.html" %}
{% block header %}
Thanks!
{% endblock %}
{% block content %}
<p>I'll use it wisely</p>
<a href="login" class="button button-big">Authorize another account!</a>
{% endblock %}

53
bitter/utils.py
View File

@ -5,12 +5,17 @@ import json
import signal
import sys
import sqlalchemy
import os
from itertools import islice
from contextlib import contextmanager
from twitter import TwitterHTTPError
from bitter.models import Following, User, ExtractorEntry, make_session
from bitter import config
logger = logging.getLogger(__name__)
@ -19,12 +24,58 @@ def signal_handler(signal, frame):
sys.exit(0)
def get_credentials_path(credfile=None):
if not credfile:
if config.CREDENTIALS:
credfile = config.CREDENTIALS
else:
raise Exception('No valid credentials file')
return os.path.expanduser(credfile)
@contextmanager
def credentials_file(credfile, *args, **kwargs):
p = get_credentials_path(credfile)
with open(p, *args, **kwargs) as f:
yield f
def iter_credentials(credfile=None):
with credentials_file(credfile) as f:
for l in f:
yield json.loads(l.strip())
def get_credentials(credfile=None, inverse=False, **kwargs):
creds = []
for i in iter_credentials(credfile):
if all(map(lambda x: i[x[0]] == x[1], kwargs.items())):
creds.append(i)
return creds
def create_credentials(credfile=None):
credfile = get_credentials_path(credfile)
with credentials_file(credfile, 'a'):
pass
def delete_credentials(credfile=None, **creds):
tokeep = get_credentials(credfile, inverse=True, **creds)
with credentials_file(credfile, 'w') as f:
for i in tokeep:
f.write(json.dumps(i))
f.write('\n')
def add_credentials(credfile=None, **creds):
exist = get_credentials(credfile, **creds)
if not exist:
with credentials_file(credfile, 'a') as f:
f.write(json.dumps(creds))
f.write('\n')
def get_users(wq, ulist, by_name=False, queue=None, max_users=100):
t = 'name' if by_name else 'uid'
logger.debug('Getting users by {}: {}'.format(t, ulist))
ilist = iter(ulist)
while True:
userslice = ",".join(islice(ilist, max_users))
userslice = ",".join(str(i) for i in islice(ilist, max_users))
if not userslice:
break
try:

96
bitter/webserver.py Normal file
View File

@ -0,0 +1,96 @@
import json
import os.path
from flask import Flask, redirect, session, url_for, request, flash, render_template
from twitter import Twitter
from twitter import OAuth as TOAuth
from flask_oauthlib.client import OAuth
from . import utils
from . import config
oauth = OAuth()
twitter = oauth.remote_app('twitter',
base_url='https://api.twitter.com/1/',
request_token_url='https://api.twitter.com/oauth/request_token',
access_token_url='https://api.twitter.com/oauth/access_token',
authorize_url='https://api.twitter.com/oauth/authenticate',
consumer_key=config.CONSUMER_KEY,
consumer_secret=config.CONSUMER_SECRET
)
app = Flask(__name__)
@twitter.tokengetter
def get_twitter_token(token=None):
return session.get('twitter_token')
@app.route('/login')
def login():
if 'twitter_token' in session:
del session['twitter_token']
return twitter.authorize(callback='/oauth-authorized')
# next=request.args.get('next') or request.referrer or None))
@app.route('/oauth-authorized')
def oauth_authorized():
resp = twitter.authorized_response()
if resp is None:
flash(u'You denied the request to sign in.')
return redirect('/sad')
token = (
resp['oauth_token'],
resp['oauth_token_secret']
)
user = resp['screen_name']
session['twitter_token'] = token
session['twitter_user'] = user
new_creds = {"token_key": token[0],
"token_secret": token[1]}
utils.delete_credentials(user=user)
utils.add_credentials(user=user,
token_key=token[0],
token_secret=token[1],
consumer_key=config.CONSUMER_KEY,
consumer_secret=config.CONSUMER_SECRET)
flash('You were signed in as %s' % resp['screen_name'])
return redirect('/thanks')
@app.route('/thanks')
def thanks():
return render_template("thanks.html")
@app.route('/sad')
def sad():
return render_template("sad.html")
@app.route('/')
def index():
# return 'Please <a href="./login">LOG IN</a> to help with my research :)'
return render_template('home.html')
@app.route('/hall')
def hall():
names = [c['user'] for c in utils.get_credentials()]
return render_template('thanks.html', names=names)
@app.route('/limits')
def limits():
creds = utils.get_credentials()
limits = {}
for c in creds:
auth = TOAuth(c['token_key'],
c['token_secret'],
c['consumer_key'],
c['consumer_secret'])
t = Twitter(auth=auth)
limits[c["user"]] = json.dumps(t.application.rate_limit_status(), indent=2)
return render_template('limits.html', limits=limits)
app.secret_key = os.environ.get('SESSION_KEY', 'bitter is cool!')
if __name__ == '__main__':
app.run()

1
requirements.txt
View File

@ -1,3 +1,4 @@
sqlalchemy
twitter
click
six

2
setup.py
View File

@ -27,7 +27,7 @@ setup(
author='J. Fernando Sanchez',
author_email='balkian@gmail.com',
url="http://balkian.com",
version="0.3",
version="0.4",
install_requires=install_reqs,
tests_require=test_reqs,
include_package_data=True,

2
tests/test_models.py
View File

@ -1,4 +1,4 @@
from unittests import TestCase
from unittest import TestCase
class TestModels(TestCase):