diff --git a/README.md b/README.md new file mode 100644 index 0000000..50a0903 --- /dev/null +++ b/README.md @@ -0,0 +1,50 @@ +#Description +There are two parts to bitter. +First of all, it is a wrapper over Python twitter that adds support for several Twitter API credentials (e.g. authorizing the same app with different user accounts). +Secondly, it is a command line tool to automate several actions (e.g. downloading user networks) using the wrapper. + +# Instructions + +In the command line: + + python -m bitter --help + +or + + bitter --help + + +Programmatically: + +```python +from bitter.crawlers import TwitterQueue +wq = TwitterQueue.from_credentials() +print(wq.users.show(user_name='balkian')) +``` + +# Credentials format + +``` +{"user": "balkian", "consumer_secret": "xxx", "consumer_key": "xxx", "token_key": "xxx", "token_secret": "xxx"} +``` + +By default, bitter uses '~/.bitter-credentials.json', but you may choose a different file: + +``` +python -m bitter -c ... +``` + +# Server +To add more users to the credentials file, you may run the builtin server, with the consumer key and secret of your app: + +``` +python -m bitter server +``` + +# Notice +Please, use according to Twitter's Terms of Service + +# TODO + +* Tests +* Docs diff --git a/bitter/cli.py b/bitter/cli.py index 07468fc..6d7c130 100644 --- a/bitter/cli.py +++ b/bitter/cli.py @@ -20,21 +20,15 @@ logger = logging.getLogger(__name__) @click.option("--verbose", is_flag=True) @click.option("--logging_level", required=False, default='WARN') @click.option("--config", required=False) -@click.option('-c', '--credentials',show_default=True, default='credentials.json') +@click.option('-c', '--credentials', show_default=True, default='~/.bitter-credentials.json') @click.pass_context def main(ctx, verbose, logging_level, config, credentials): logging.basicConfig(level=getattr(logging, logging_level)) ctx.obj = {} ctx.obj['VERBOSE'] = verbose ctx.obj['CONFIG'] = config - if os.path.isfile(credentials): - ctx.obj['CREDENTIALS'] = credentials - else: - global_file = os.path.expanduser('~/.bitter-credentials.json') - if os.path.isfile(global_file): - ctx.obj['CREDENTIALS'] = global_file - else: - raise Exception('You need to provide a valid credentials file') + ctx.obj['CREDENTIALS'] = credentials + utils.create_credentials(credentials) @main.group() @click.pass_context @@ -46,8 +40,7 @@ def tweet(ctx): @click.pass_context def get_tweet(ctx, tweetid): wq = crawlers.TwitterQueue.from_credentials(ctx.obj['CREDENTIALS']) - c = wq.next() - t = crawlers.get_tweet(c.client, tweetid) + t = utils.get_tweet(wq, tweetid) print(json.dumps(t, indent=2)) @@ -320,5 +313,17 @@ def get_limits(ctx, url): else: print(json.dumps(resp, indent=2)) +@main.command('server') +@click.argument('CONSUMER_KEY', required=True) +@click.argument('CONSUMER_SECRET', required=True) +@click.pass_context +def run_server(ctx, consumer_key, consumer_secret): + from . import config + config.CONSUMER_KEY = consumer_key + config.CONSUMER_SECRET = consumer_secret + from .webserver import app + app.run() + + if __name__ == '__main__': main() diff --git a/bitter/config.py b/bitter/config.py new file mode 100644 index 0000000..c35066e --- /dev/null +++ b/bitter/config.py @@ -0,0 +1,13 @@ +''' +Common configuration for other modules. +It is not elegant, but it works with flask and the oauth decorators. + +Using this module allows you to change the config before loading any other module. +E.g.: + + import bitter.config as c + c.CREDENTIALS="/tmp/credentials" + from bitter.webserver import app + app.run() +''' +CREDENTIALS = '~/.bitter-credentials.json' diff --git a/bitter/crawlers.py b/bitter/crawlers.py index ed0570a..fc2122b 100644 --- a/bitter/crawlers.py +++ b/bitter/crawlers.py @@ -8,6 +8,8 @@ logger = logging.getLogger(__name__) from twitter import * from collections import OrderedDict +from . import utils +from . import config class AttrToFunc(object): @@ -102,17 +104,15 @@ class TwitterQueue(AttrToFunc): return self.next().client @classmethod - def from_credentials(self, cred_file): + def from_credentials(self, cred_file=None): wq = TwitterQueue() - with open(cred_file) as f: - for line in f: - cred = json.loads(line) - c = Twitter(auth=OAuth(cred['token_key'], - cred['token_secret'], - cred['consumer_key'], - cred['consumer_secret'])) - wq.ready(TwitterWorker(cred["user"], c)) + for cred in utils.get_credentials(cred_file): + c = Twitter(auth=OAuth(cred['token_key'], + cred['token_secret'], + cred['consumer_key'], + cred['consumer_secret'])) + wq.ready(TwitterWorker(cred["user"], c)) return wq def _next(self): diff --git a/bitter/static/css/style.css b/bitter/static/css/style.css new file mode 100644 index 0000000..8586c33 --- /dev/null +++ b/bitter/static/css/style.css @@ -0,0 +1,50 @@ +@charset 'UTF-8'; + +html { + display: block; + margin: auto; + width: 100%; + background-color: black; + color: white; + font-size: 1.5em; +} +.wrapper { + margin: 0 auto; + width: 80%; + text-align: center; +} + +.me { + width: 15em; + border-radius: 150px; + -webkit-border-radius: 150px; + -moz-border-radius: 150px; + +} + +.button { + background-color: black; + color: white; + padding: 0.5em; + font-size: 1.5em; + border: 3px solid white; + margin-top: 1em; + display: inline-block; + border-radius: 1em; + text-decoration: none; + font-weight: bold; +} + +.button:hover { + color: black; + background-color: white; +} + +.limits { + text-align: left; +} + +.jsonlimits { + overflow: auto; + max-height: 5em; +} \ No newline at end of file diff --git a/bitter/static/images/me.jpg b/bitter/static/images/me.jpg new file mode 100644 index 0000000..3f1bd75 Binary files /dev/null and b/bitter/static/images/me.jpg differ diff --git a/bitter/static/index.html b/bitter/static/index.html new file mode 100644 index 0000000..92c729e --- /dev/null +++ b/bitter/static/index.html @@ -0,0 +1,241 @@ + + + + + Miniport by HTML5 UP + + + + + + + + + + + + + + + + + + +
+ +
+ + +
+
+
+

I design and build amazing things.

+ Odio turpis amet sed consequat eget posuere consequat. +
+
+
+
+
+ +

Consequat lorem

+

Ornare nulla proin odio consequat sapien vestibulum ipsum primis sed amet consequat lorem dolore.

+
+
+
+
+ +

Lorem dolor tempus

+

Ornare nulla proin odio consequat sapien vestibulum ipsum primis sed amet consequat lorem dolore.

+
+
+
+
+ +

Feugiat posuere

+

Ornare nulla proin odio consequat sapien vestibulum ipsum primis sed amet consequat lorem dolore.

+
+
+
+
+ +
+
+ + +
+
+
+

Awesome work makes happy clients.

+ Proin odio consequat sapien vestibulum ipsum primis sed amet consequat lorem dolore feugiat lorem ipsum dolore. +
+
+
+
+
+
+
+
+ +
+
+ +
+
+
+ +

Lorem ipsum

+

Ornare nulla proin odio consequat.

+
+
+
+
+
+ +
+
+ +
+
+ +
+
+
+ +
+
+ + +
+ +
+ + + + \ No newline at end of file diff --git a/bitter/templates/base.html b/bitter/templates/base.html new file mode 100644 index 0000000..3c1bb98 --- /dev/null +++ b/bitter/templates/base.html @@ -0,0 +1,36 @@ + + + + + Bitter's Login + + + + + + + + + + + + +
+ +
+ {% block header %} + {% endblock %} +
+
+ {% block content %} + {% endblock %} +
+
+ + + + diff --git a/bitter/templates/home.html b/bitter/templates/home.html new file mode 100644 index 0000000..73d71ca --- /dev/null +++ b/bitter/templates/home.html @@ -0,0 +1,10 @@ +{% extends "base.html" %} +{% block header %} +

Hi. I'm a researcher.

+{% endblock %} +{% block content %} +

Getting data from Twitter is hard, due to the limits imposed by the Twitter API

+

By logging in with my app you help me get more data for my research.

+

I will not use your personal information in any way.

+Log in! +{% endblock %} diff --git a/bitter/templates/limits.html b/bitter/templates/limits.html new file mode 100644 index 0000000..d6397a3 --- /dev/null +++ b/bitter/templates/limits.html @@ -0,0 +1,11 @@ +{% extends "base.html" %} +{% block content %} +{% if limits | length %} +
    + {% for limit in limits %} +
  • {{ limit }} 
    {{ limits[limit] }}
    • + {% endfor %} +
+{% endif %} + +{% endblock %} diff --git a/bitter/templates/sad.html b/bitter/templates/sad.html new file mode 100644 index 0000000..6027e1a --- /dev/null +++ b/bitter/templates/sad.html @@ -0,0 +1,8 @@ +{% extends "base.html" %} +{% block header %} +

You don't wanna authorize me? :(

+{% endblock %} +{% block content %} +

It's ok, but come back if you change your mind...

+ +{% endblock %} diff --git a/bitter/templates/thanks.html b/bitter/templates/thanks.html new file mode 100644 index 0000000..b09ae1c --- /dev/null +++ b/bitter/templates/thanks.html @@ -0,0 +1,10 @@ +{% extends "base.html" %} + +{% block header %} +Thanks! +{% endblock %} +{% block content %} +

I'll use it wisely

+ Authorize another account! +{% endblock %} + diff --git a/bitter/utils.py b/bitter/utils.py index 78721ee..805764b 100644 --- a/bitter/utils.py +++ b/bitter/utils.py @@ -5,12 +5,17 @@ import json import signal import sys import sqlalchemy +import os from itertools import islice +from contextlib import contextmanager + from twitter import TwitterHTTPError from bitter.models import Following, User, ExtractorEntry, make_session +from bitter import config + logger = logging.getLogger(__name__) @@ -19,12 +24,58 @@ def signal_handler(signal, frame): sys.exit(0) +def get_credentials_path(credfile=None): + if not credfile: + if config.CREDENTIALS: + credfile = config.CREDENTIALS + else: + raise Exception('No valid credentials file') + return os.path.expanduser(credfile) + +@contextmanager +def credentials_file(credfile, *args, **kwargs): + p = get_credentials_path(credfile) + with open(p, *args, **kwargs) as f: + yield f + +def iter_credentials(credfile=None): + with credentials_file(credfile) as f: + for l in f: + yield json.loads(l.strip()) + +def get_credentials(credfile=None, inverse=False, **kwargs): + creds = [] + for i in iter_credentials(credfile): + if all(map(lambda x: i[x[0]] == x[1], kwargs.items())): + creds.append(i) + return creds + +def create_credentials(credfile=None): + credfile = get_credentials_path(credfile) + with credentials_file(credfile, 'a'): + pass + +def delete_credentials(credfile=None, **creds): + tokeep = get_credentials(credfile, inverse=True, **creds) + with credentials_file(credfile, 'w') as f: + for i in tokeep: + f.write(json.dumps(i)) + f.write('\n') + +def add_credentials(credfile=None, **creds): + exist = get_credentials(credfile, **creds) + if not exist: + with credentials_file(credfile, 'a') as f: + f.write(json.dumps(creds)) + f.write('\n') + + def get_users(wq, ulist, by_name=False, queue=None, max_users=100): t = 'name' if by_name else 'uid' logger.debug('Getting users by {}: {}'.format(t, ulist)) ilist = iter(ulist) while True: - userslice = ",".join(islice(ilist, max_users)) + userslice = ",".join(str(i) for i in islice(ilist, max_users)) if not userslice: break try: @@ -48,7 +99,7 @@ def get_users(wq, ulist, by_name=False, queue=None, max_users=100): def trim_user(user): if 'status' in user: - del user['status'] + del user['status'] if 'follow_request_sent' in user: del user['follow_request_sent'] if 'created_at' in user: @@ -64,7 +115,7 @@ def add_user(session, user, enqueue=False): olduser = session.query(User).filter(User.id==user['id']) if olduser: olduser.delete() - user = User(**user) + user = User(**user) session.add(user) if extract: logging.debug('Adding entry') @@ -76,12 +127,12 @@ def add_user(session, user, enqueue=False): entry.pending = True entry.cursor = -1 session.commit() - + # TODO: adapt to the crawler def extract(wq, recursive=False, user=None, initfile=None, dburi=None, extractor_name=None): signal.signal(signal.SIGINT, signal_handler) - + w = wq.next() if not dburi: dburi = 'sqlite:///%s.db' % extractor_name @@ -99,7 +150,7 @@ def extract(wq, recursive=False, user=None, initfile=None, dburi=None, extractor except ValueError: logger.info("Added screen_name") screen_names.append(user.split('@')[-1]) - + if user: classify_user(user) @@ -123,7 +174,7 @@ def extract(wq, recursive=False, user=None, initfile=None, dburi=None, extractor total_users = session.query(sqlalchemy.func.count(User.id)).scalar() logging.info('Total users: {}'.format(total_users)) def pending_entries(): - pending = session.query(ExtractorEntry).filter(ExtractorEntry.pending == True).count() + pending = session.query(ExtractorEntry).filter(ExtractorEntry.pending == True).count() logging.info('Pending: {}'.format(pending)) return pending @@ -192,7 +243,7 @@ def extract(wq, recursive=False, user=None, initfile=None, dburi=None, extractor session.add(candidate) session.commit() - + sys.stdout.flush() diff --git a/bitter/webserver.py b/bitter/webserver.py new file mode 100644 index 0000000..ec59016 --- /dev/null +++ b/bitter/webserver.py @@ -0,0 +1,96 @@ +import json +import os.path +from flask import Flask, redirect, session, url_for, request, flash, render_template +from twitter import Twitter +from twitter import OAuth as TOAuth + +from flask_oauthlib.client import OAuth + +from . import utils +from . import config + +oauth = OAuth() +twitter = oauth.remote_app('twitter', + base_url='https://api.twitter.com/1/', + request_token_url='https://api.twitter.com/oauth/request_token', + access_token_url='https://api.twitter.com/oauth/access_token', + authorize_url='https://api.twitter.com/oauth/authenticate', + consumer_key=config.CONSUMER_KEY, + consumer_secret=config.CONSUMER_SECRET + ) + +app = Flask(__name__) + +@twitter.tokengetter +def get_twitter_token(token=None): + return session.get('twitter_token') + + +@app.route('/login') +def login(): + if 'twitter_token' in session: + del session['twitter_token'] + return twitter.authorize(callback='/oauth-authorized') +# next=request.args.get('next') or request.referrer or None)) + +@app.route('/oauth-authorized') +def oauth_authorized(): + resp = twitter.authorized_response() + if resp is None: + flash(u'You denied the request to sign in.') + return redirect('/sad') + token = ( + resp['oauth_token'], + resp['oauth_token_secret'] + ) + user = resp['screen_name'] + session['twitter_token'] = token + session['twitter_user'] = user + new_creds = {"token_key": token[0], + "token_secret": token[1]} + + utils.delete_credentials(user=user) + utils.add_credentials(user=user, + token_key=token[0], + token_secret=token[1], + consumer_key=config.CONSUMER_KEY, + consumer_secret=config.CONSUMER_SECRET) + flash('You were signed in as %s' % resp['screen_name']) + return redirect('/thanks') + +@app.route('/thanks') +def thanks(): + return render_template("thanks.html") + +@app.route('/sad') +def sad(): + return render_template("sad.html") + +@app.route('/') +def index(): +# return 'Please LOG IN to help with my research :)' + return render_template('home.html') + +@app.route('/hall') +def hall(): + names = [c['user'] for c in utils.get_credentials()] + return render_template('thanks.html', names=names) + +@app.route('/limits') +def limits(): + creds = utils.get_credentials() + limits = {} + for c in creds: + auth = TOAuth(c['token_key'], + c['token_secret'], + c['consumer_key'], + c['consumer_secret']) + t = Twitter(auth=auth) + limits[c["user"]] = json.dumps(t.application.rate_limit_status(), indent=2) + + return render_template('limits.html', limits=limits) + +app.secret_key = os.environ.get('SESSION_KEY', 'bitter is cool!') + +if __name__ == '__main__': + app.run() diff --git a/requirements.txt b/requirements.txt index 63537f8..fa6c088 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ sqlalchemy twitter click +six diff --git a/setup.py b/setup.py index a410995..2e9bf94 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ setup( author='J. Fernando Sanchez', author_email='balkian@gmail.com', url="http://balkian.com", - version="0.3", + version="0.4", install_requires=install_reqs, tests_require=test_reqs, include_package_data=True, diff --git a/tests/test_models.py b/tests/test_models.py index 3020a54..c87a1de 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,4 +1,4 @@ -from unittests import TestCase +from unittest import TestCase class TestModels(TestCase):