mirror of
https://github.com/balkian/bitter.git
synced 2024-12-22 00:18:12 +00:00
Added webserver
This commit is contained in:
J. Fernando Sánchez
parent
97028e38b1
commit
17f589c710
50
README.md
Normal file
50
README.md
Normal file
@ -0,0 +1,50 @@
|
|||||||
|
#Description
|
||||||
|
There are two parts to bitter.
|
||||||
|
First of all, it is a wrapper over Python twitter that adds support for several Twitter API credentials (e.g. authorizing the same app with different user accounts).
|
||||||
|
Secondly, it is a command line tool to automate several actions (e.g. downloading user networks) using the wrapper.
|
||||||
|
|
||||||
|
# Instructions
|
||||||
|
|
||||||
|
In the command line:
|
||||||
|
|
||||||
|
python -m bitter --help
|
||||||
|
|
||||||
|
or
|
||||||
|
|
||||||
|
bitter --help
|
||||||
|
|
||||||
|
|
||||||
|
Programmatically:
|
||||||
|
|
||||||
|
```python
|
||||||
|
from bitter.crawlers import TwitterQueue
|
||||||
|
wq = TwitterQueue.from_credentials()
|
||||||
|
print(wq.users.show(user_name='balkian'))
|
||||||
|
```
|
||||||
|
|
||||||
|
# Credentials format
|
||||||
|
|
||||||
|
```
|
||||||
|
{"user": "balkian", "consumer_secret": "xxx", "consumer_key": "xxx", "token_key": "xxx", "token_secret": "xxx"}
|
||||||
|
```
|
||||||
|
|
||||||
|
By default, bitter uses '~/.bitter-credentials.json', but you may choose a different file:
|
||||||
|
|
||||||
|
```
|
||||||
|
python -m bitter -c <credentials_file> ...
|
||||||
|
```
|
||||||
|
|
||||||
|
# Server
|
||||||
|
To add more users to the credentials file, you may run the builtin server, with the consumer key and secret of your app:
|
||||||
|
|
||||||
|
```
|
||||||
|
python -m bitter server <consumer_key> <consumer_secret>
|
||||||
|
```
|
||||||
|
|
||||||
|
# Notice
|
||||||
|
Please, use according to Twitter's Terms of Service
|
||||||
|
|
||||||
|
# TODO
|
||||||
|
|
||||||
|
* Tests
|
||||||
|
* Docs
|
||||||
@ -20,21 +20,15 @@ logger = logging.getLogger(__name__)
|
|||||||
@click.option("--verbose", is_flag=True)
|
@click.option("--verbose", is_flag=True)
|
||||||
@click.option("--logging_level", required=False, default='WARN')
|
@click.option("--logging_level", required=False, default='WARN')
|
||||||
@click.option("--config", required=False)
|
@click.option("--config", required=False)
|
||||||
@click.option('-c', '--credentials',show_default=True, default='credentials.json')
|
@click.option('-c', '--credentials', show_default=True, default='~/.bitter-credentials.json')
|
||||||
@click.pass_context
|
@click.pass_context
|
||||||
def main(ctx, verbose, logging_level, config, credentials):
|
def main(ctx, verbose, logging_level, config, credentials):
|
||||||
logging.basicConfig(level=getattr(logging, logging_level))
|
logging.basicConfig(level=getattr(logging, logging_level))
|
||||||
ctx.obj = {}
|
ctx.obj = {}
|
||||||
ctx.obj['VERBOSE'] = verbose
|
ctx.obj['VERBOSE'] = verbose
|
||||||
ctx.obj['CONFIG'] = config
|
ctx.obj['CONFIG'] = config
|
||||||
if os.path.isfile(credentials):
|
ctx.obj['CREDENTIALS'] = credentials
|
||||||
ctx.obj['CREDENTIALS'] = credentials
|
utils.create_credentials(credentials)
|
||||||
else:
|
|
||||||
global_file = os.path.expanduser('~/.bitter-credentials.json')
|
|
||||||
if os.path.isfile(global_file):
|
|
||||||
ctx.obj['CREDENTIALS'] = global_file
|
|
||||||
else:
|
|
||||||
raise Exception('You need to provide a valid credentials file')
|
|
||||||
|
|
||||||
@main.group()
|
@main.group()
|
||||||
@click.pass_context
|
@click.pass_context
|
||||||
@ -46,8 +40,7 @@ def tweet(ctx):
|
|||||||
@click.pass_context
|
@click.pass_context
|
||||||
def get_tweet(ctx, tweetid):
|
def get_tweet(ctx, tweetid):
|
||||||
wq = crawlers.TwitterQueue.from_credentials(ctx.obj['CREDENTIALS'])
|
wq = crawlers.TwitterQueue.from_credentials(ctx.obj['CREDENTIALS'])
|
||||||
c = wq.next()
|
t = utils.get_tweet(wq, tweetid)
|
||||||
t = crawlers.get_tweet(c.client, tweetid)
|
|
||||||
print(json.dumps(t, indent=2))
|
print(json.dumps(t, indent=2))
|
||||||
|
|
||||||
|
|
||||||
@ -320,5 +313,17 @@ def get_limits(ctx, url):
|
|||||||
else:
|
else:
|
||||||
print(json.dumps(resp, indent=2))
|
print(json.dumps(resp, indent=2))
|
||||||
|
|
||||||
|
@main.command('server')
|
||||||
|
@click.argument('CONSUMER_KEY', required=True)
|
||||||
|
@click.argument('CONSUMER_SECRET', required=True)
|
||||||
|
@click.pass_context
|
||||||
|
def run_server(ctx, consumer_key, consumer_secret):
|
||||||
|
from . import config
|
||||||
|
config.CONSUMER_KEY = consumer_key
|
||||||
|
config.CONSUMER_SECRET = consumer_secret
|
||||||
|
from .webserver import app
|
||||||
|
app.run()
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
main()
|
main()
|
||||||
|
|||||||
13
bitter/config.py
Normal file
13
bitter/config.py
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
'''
|
||||||
|
Common configuration for other modules.
|
||||||
|
It is not elegant, but it works with flask and the oauth decorators.
|
||||||
|
|
||||||
|
Using this module allows you to change the config before loading any other module.
|
||||||
|
E.g.:
|
||||||
|
|
||||||
|
import bitter.config as c
|
||||||
|
c.CREDENTIALS="/tmp/credentials"
|
||||||
|
from bitter.webserver import app
|
||||||
|
app.run()
|
||||||
|
'''
|
||||||
|
CREDENTIALS = '~/.bitter-credentials.json'
|
||||||
@ -8,6 +8,8 @@ logger = logging.getLogger(__name__)
|
|||||||
|
|
||||||
from twitter import *
|
from twitter import *
|
||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
|
from . import utils
|
||||||
|
from . import config
|
||||||
|
|
||||||
|
|
||||||
class AttrToFunc(object):
|
class AttrToFunc(object):
|
||||||
@ -102,17 +104,15 @@ class TwitterQueue(AttrToFunc):
|
|||||||
return self.next().client
|
return self.next().client
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_credentials(self, cred_file):
|
def from_credentials(self, cred_file=None):
|
||||||
wq = TwitterQueue()
|
wq = TwitterQueue()
|
||||||
|
|
||||||
with open(cred_file) as f:
|
for cred in utils.get_credentials(cred_file):
|
||||||
for line in f:
|
c = Twitter(auth=OAuth(cred['token_key'],
|
||||||
cred = json.loads(line)
|
cred['token_secret'],
|
||||||
c = Twitter(auth=OAuth(cred['token_key'],
|
cred['consumer_key'],
|
||||||
cred['token_secret'],
|
cred['consumer_secret']))
|
||||||
cred['consumer_key'],
|
wq.ready(TwitterWorker(cred["user"], c))
|
||||||
cred['consumer_secret']))
|
|
||||||
wq.ready(TwitterWorker(cred["user"], c))
|
|
||||||
return wq
|
return wq
|
||||||
|
|
||||||
def _next(self):
|
def _next(self):
|
||||||
|
|||||||
50
bitter/static/css/style.css
Normal file
50
bitter/static/css/style.css
Normal file
@ -0,0 +1,50 @@
|
|||||||
|
@charset 'UTF-8';
|
||||||
|
|
||||||
|
html {
|
||||||
|
display: block;
|
||||||
|
margin: auto;
|
||||||
|
width: 100%;
|
||||||
|
background-color: black;
|
||||||
|
color: white;
|
||||||
|
font-size: 1.5em;
|
||||||
|
}
|
||||||
|
.wrapper {
|
||||||
|
margin: 0 auto;
|
||||||
|
width: 80%;
|
||||||
|
text-align: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
.me {
|
||||||
|
width: 15em;
|
||||||
|
border-radius: 150px;
|
||||||
|
-webkit-border-radius: 150px;
|
||||||
|
-moz-border-radius: 150px;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
.button {
|
||||||
|
background-color: black;
|
||||||
|
color: white;
|
||||||
|
padding: 0.5em;
|
||||||
|
font-size: 1.5em;
|
||||||
|
border: 3px solid white;
|
||||||
|
margin-top: 1em;
|
||||||
|
display: inline-block;
|
||||||
|
border-radius: 1em;
|
||||||
|
text-decoration: none;
|
||||||
|
font-weight: bold;
|
||||||
|
}
|
||||||
|
|
||||||
|
.button:hover {
|
||||||
|
color: black;
|
||||||
|
background-color: white;
|
||||||
|
}
|
||||||
|
|
||||||
|
.limits {
|
||||||
|
text-align: left;
|
||||||
|
}
|
||||||
|
|
||||||
|
.jsonlimits {
|
||||||
|
overflow: auto;
|
||||||
|
max-height: 5em;
|
||||||
|
}
|
||||||
BIN
bitter/static/images/me.jpg
Normal file
BIN
bitter/static/images/me.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 104 KiB |
241
bitter/static/index.html
Normal file
241
bitter/static/index.html
Normal file
@ -0,0 +1,241 @@
|
|||||||
|
<!DOCTYPE HTML>
|
||||||
|
<!--
|
||||||
|
Miniport 2.5 by HTML5 UP
|
||||||
|
html5up.net | @n33co
|
||||||
|
Free for personal and commercial use under the CCA 3.0 license (html5up.net/license)
|
||||||
|
-->
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<title>Miniport by HTML5 UP</title>
|
||||||
|
<meta http-equiv="content-type" content="text/html; charset=utf-8" />
|
||||||
|
<meta name="description" content="" />
|
||||||
|
<meta name="keywords" content="" />
|
||||||
|
<link href="http://fonts.googleapis.com/css?family=Open+Sans:300,600,700" rel="stylesheet" />
|
||||||
|
<script src="js/jquery.min.js"></script>
|
||||||
|
<script src="js/config.js"></script>
|
||||||
|
<script src="js/skel.min.js"></script>
|
||||||
|
<noscript>
|
||||||
|
<link rel="stylesheet" href="css/skel-noscript.css" />
|
||||||
|
<link rel="stylesheet" href="css/style.css" />
|
||||||
|
<link rel="stylesheet" href="css/style-desktop.css" />
|
||||||
|
</noscript>
|
||||||
|
<!--[if lte IE 9]><link rel="stylesheet" href="css/ie9.css" /><![endif]-->
|
||||||
|
<!--[if lte IE 8]><script src="js/html5shiv.js"></script><link rel="stylesheet" href="css/ie8.css" /><![endif]-->
|
||||||
|
<!--[if lte IE 7]><link rel="stylesheet" href="css/ie7.css" /><![endif]-->
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
|
||||||
|
<!-- Nav -->
|
||||||
|
<nav id="nav">
|
||||||
|
<ul class="container">
|
||||||
|
<li><a href="#top">Top</a></li>
|
||||||
|
<li><a href="#work">Work</a></li>
|
||||||
|
<li><a href="#portfolio">Portfolio</a></li>
|
||||||
|
<li><a href="#contact">Contact</a></li>
|
||||||
|
</ul>
|
||||||
|
</nav>
|
||||||
|
|
||||||
|
<!-- Home -->
|
||||||
|
<div class="wrapper wrapper-style1 wrapper-first">
|
||||||
|
<article class="container" id="top">
|
||||||
|
<div class="row">
|
||||||
|
<div class="4u">
|
||||||
|
<span class="me image image-full"><img src="images/me.jpg" alt="" /></span>
|
||||||
|
</div>
|
||||||
|
<div class="8u">
|
||||||
|
<header>
|
||||||
|
<h1>Hi. I'm <strong>Jane Doe</strong>.</h1>
|
||||||
|
</header>
|
||||||
|
<p>And this is <strong>Miniport</strong>, a free, fully responsive HTML5 site template designed by <a href="http://n33.co/">AJ</a> for <a href="http://html5up.net/">HTML5 UP</a> & released under the <a href="http://html5up.net/license/">CCA license</a>.</p>
|
||||||
|
<a href="#work" class="button button-big">Learn about what I do</a>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</article>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Work -->
|
||||||
|
<div class="wrapper wrapper-style2">
|
||||||
|
<article id="work">
|
||||||
|
<header>
|
||||||
|
<h2>I design and build amazing things.</h2>
|
||||||
|
<span>Odio turpis amet sed consequat eget posuere consequat.</span>
|
||||||
|
</header>
|
||||||
|
<div class="container">
|
||||||
|
<div class="row">
|
||||||
|
<div class="4u">
|
||||||
|
<section class="box box-style1">
|
||||||
|
<span class="fa featured fa-comments-o"></span>
|
||||||
|
<h3>Consequat lorem</h3>
|
||||||
|
<p>Ornare nulla proin odio consequat sapien vestibulum ipsum primis sed amet consequat lorem dolore.</p>
|
||||||
|
</section>
|
||||||
|
</div>
|
||||||
|
<div class="4u">
|
||||||
|
<section class="box box-style1">
|
||||||
|
<span class="fa featured fa-file-o"></span>
|
||||||
|
<h3>Lorem dolor tempus</h3>
|
||||||
|
<p>Ornare nulla proin odio consequat sapien vestibulum ipsum primis sed amet consequat lorem dolore.</p>
|
||||||
|
</section>
|
||||||
|
</div>
|
||||||
|
<div class="4u">
|
||||||
|
<section class="box box-style1">
|
||||||
|
<span class="fa featured fa-thumbs-o-up"></span>
|
||||||
|
<h3>Feugiat posuere</h3>
|
||||||
|
<p>Ornare nulla proin odio consequat sapien vestibulum ipsum primis sed amet consequat lorem dolore.</p>
|
||||||
|
</section>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<footer>
|
||||||
|
<p>Lorem ipsum dolor sit sapien vestibulum ipsum primis?</p>
|
||||||
|
<a href="#portfolio" class="button button-big">See some of my recent work</a>
|
||||||
|
</footer>
|
||||||
|
</article>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Portfolio -->
|
||||||
|
<div class="wrapper wrapper-style3">
|
||||||
|
<article id="portfolio">
|
||||||
|
<header>
|
||||||
|
<h2>Awesome work makes happy clients.</h2>
|
||||||
|
<span>Proin odio consequat sapien vestibulum ipsum primis sed amet consequat lorem dolore feugiat lorem ipsum dolore.</span>
|
||||||
|
</header>
|
||||||
|
<div class="container">
|
||||||
|
<div class="row">
|
||||||
|
<div class="12u">
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="row">
|
||||||
|
<div class="4u">
|
||||||
|
<article class="box box-style2">
|
||||||
|
<a href="http://flypixel.com/generic-smartphone/8949517882265310" class="image image-full"><img src="images/portfolio01.jpg" alt="" /></a>
|
||||||
|
<h3><a href="http://flypixel.com/generic-smartphone/8949517882265310">Magna feugiat</a></h3>
|
||||||
|
<p>Ornare nulla proin odio consequat.</p>
|
||||||
|
</article>
|
||||||
|
</div>
|
||||||
|
<div class="4u">
|
||||||
|
<article class="box box-style2">
|
||||||
|
<a href="http://flypixel.com/n33" class="image image-full"><img src="images/portfolio02.jpg" alt="" /></a>
|
||||||
|
<h3><a href="http://flypixel.com/n33">Veroeros primis</a></h3>
|
||||||
|
<p>Ornare nulla proin odio consequat.</p>
|
||||||
|
</article>
|
||||||
|
</div>
|
||||||
|
<div class="4u">
|
||||||
|
<article class="box box-style2">
|
||||||
|
<a href="http://flypixel.com/wood-ui-kit/3574765984616310" class="image image-full"><img src="images/portfolio03.jpg" alt="" /></a>
|
||||||
|
<h3><a href="http://flypixel.com/wood-ui-kit/3574765984616310">Lorem ipsum</a></h3>
|
||||||
|
<p>Ornare nulla proin odio consequat.</p>
|
||||||
|
</article>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="row">
|
||||||
|
<div class="4u">
|
||||||
|
<article class="box box-style2">
|
||||||
|
<a href="http://flypixel.com/n33-pattern-set-1/3522389001865317" class="image image-full"><img src="images/portfolio04.jpg" alt="" /></a>
|
||||||
|
<h3><a href="http://flypixel.com/n33-pattern-set-1/3522389001865317">Tempus dolore</a></h3>
|
||||||
|
<p>Ornare nulla proin odio consequat.</p>
|
||||||
|
</article>
|
||||||
|
</div>
|
||||||
|
<div class="4u">
|
||||||
|
<article class="box box-style2">
|
||||||
|
<a href="http://flypixel.com/cityscape/9803996277226316" class="image image-full"><img src="images/portfolio05.jpg" alt="" /></a>
|
||||||
|
<h3><a href="http://flypixel.com/cityscape/9803996277226316">Feugiat aliquam</a></h3>
|
||||||
|
<p>Ornare nulla proin odio consequat.</p>
|
||||||
|
</article>
|
||||||
|
</div>
|
||||||
|
<div class="4u">
|
||||||
|
<article class="box box-style2">
|
||||||
|
<a href="http://flypixel.com/n33" class="image image-full"><img src="images/portfolio06.jpg" alt="" /></a>
|
||||||
|
<h3><a href="http://flypixel.com/n33">Sed amet ornare</a></h3>
|
||||||
|
<p>Ornare nulla proin odio consequat.</p>
|
||||||
|
</article>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<footer>
|
||||||
|
<p>Lorem ipsum dolor sit sapien vestibulum ipsum primis?</p>
|
||||||
|
<a href="#contact" class="button button-big">Get in touch with me</a>
|
||||||
|
</footer>
|
||||||
|
</article>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Contact -->
|
||||||
|
<div class="wrapper wrapper-style4">
|
||||||
|
<article id="contact" class="container small">
|
||||||
|
<header>
|
||||||
|
<h2>Want to hire me? Get in touch!</h2>
|
||||||
|
<span>Ornare nulla proin odio consequat sapien vestibulum ipsum sed lorem.</span>
|
||||||
|
</header>
|
||||||
|
<div>
|
||||||
|
<div class="row">
|
||||||
|
<div class="12u">
|
||||||
|
<form method="post" action="#">
|
||||||
|
<div>
|
||||||
|
<div class="row half">
|
||||||
|
<div class="6u">
|
||||||
|
<input type="text" name="name" id="name" placeholder="Name" />
|
||||||
|
</div>
|
||||||
|
<div class="6u">
|
||||||
|
<input type="text" name="email" id="email" placeholder="Email" />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="row half">
|
||||||
|
<div class="12u">
|
||||||
|
<input type="text" name="subject" id="subject" placeholder="Subject" />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="row half">
|
||||||
|
<div class="12u">
|
||||||
|
<textarea name="message" id="message" placeholder="Message"></textarea>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="row">
|
||||||
|
<div class="12u">
|
||||||
|
<a href="#" class="button form-button-submit">Send Message</a>
|
||||||
|
<a href="#" class="button button-alt form-button-reset">Clear Form</a>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="row">
|
||||||
|
<div class="12u">
|
||||||
|
<hr />
|
||||||
|
<h3>Find me on ...</h3>
|
||||||
|
<ul class="social">
|
||||||
|
<li class="twitter"><a href="http://twitter.com/n33co" class="fa fa-twitter"><span>Twitter</span></a></li>
|
||||||
|
<li class="facebook"><a href="#" class="fa fa-facebook"><span>Facebook</span></a></li>
|
||||||
|
<li class="dribbble"><a href="http://dribbble.com/n33" class="fa fa-dribbble"><span>Dribbble</span></a></li>
|
||||||
|
<li class="linkedin"><a href="#" class="fa fa-linkedin"><span>LinkedIn</span></a></li>
|
||||||
|
<li class="tumblr"><a href="#" class="fa fa-tumblr"><span>Tumblr</span></a></li>
|
||||||
|
<li class="googleplus"><a href="#" class="fa fa-google-plus"><span>Google+</span></a></li>
|
||||||
|
<li class="github"><a href="http://github.com/n33" class="fa fa-github"><span>Github</span></a></li>
|
||||||
|
<!--
|
||||||
|
<li class="rss"><a href="#" class="fa fa-rss"><span>RSS</span></a></li>
|
||||||
|
<li class="instagram"><a href="#" class="fa fa-instagram"><span>Instagram</span></a></li>
|
||||||
|
<li class="foursquare"><a href="#" class="fa fa-foursquare"><span>Foursquare</span></a></li>
|
||||||
|
<li class="skype"><a href="#" class="fa fa-skype"><span>Skype</span></a></li>
|
||||||
|
<li class="soundcloud"><a href="#" class="fa fa-soundcloud"><span>Soundcloud</span></a></li>
|
||||||
|
<li class="youtube"><a href="#" class="fa fa-youtube"><span>YouTube</span></a></li>
|
||||||
|
<li class="blogger"><a href="#" class="fa fa-blogger"><span>Blogger</span></a></li>
|
||||||
|
<li class="flickr"><a href="#" class="fa fa-flickr"><span>Flickr</span></a></li>
|
||||||
|
<li class="vimeo"><a href="#" class="fa fa-vimeo"><span>Vimeo</span></a></li>
|
||||||
|
-->
|
||||||
|
</ul>
|
||||||
|
<hr />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<footer>
|
||||||
|
<ul id="copyright">
|
||||||
|
<li>© 2013 Jane Doe</li>
|
||||||
|
<li>Images: <a href="http://fotogrph.com">fotogrph</a></li>
|
||||||
|
<li>Design: <a href="http://html5up.net/">HTML5 UP</a></li>
|
||||||
|
</ul>
|
||||||
|
</footer>
|
||||||
|
</article>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
36
bitter/templates/base.html
Normal file
36
bitter/templates/base.html
Normal file
@ -0,0 +1,36 @@
|
|||||||
|
<!DOCTYPE HTML>
|
||||||
|
<!--
|
||||||
|
Miniport 2.5 by HTML5 UP
|
||||||
|
html5up.net | @n33co
|
||||||
|
Free for personal and commercial use under the CCA 3.0 license (html5up.net/license)
|
||||||
|
-->
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<title>Bitter's Login</title>
|
||||||
|
<meta http-equiv="content-type" content="text/html; charset=utf-8" />
|
||||||
|
<meta name="description" content="" />
|
||||||
|
<meta name="keywords" content="" />
|
||||||
|
<link rel="stylesheet" href="static/css/style.css" />
|
||||||
|
<!--[if lte IE 9]><link rel="stylesheet" href="static/css/ie9.css" /><![endif]-->
|
||||||
|
<!--[if lte IE 8]><script src="static/js/html5shiv.js"></script><link rel="stylesheet" href="static/css/ie8.css" /><![endif]-->
|
||||||
|
<!--[if lte IE 7]><link rel="stylesheet" href="static/css/ie7.css" /><![endif]-->
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
|
||||||
|
<!-- Nav -->
|
||||||
|
<!-- Home -->
|
||||||
|
<div class="wrapper">
|
||||||
|
<img class="me" src="static/images/me.jpg" alt="" />
|
||||||
|
<header>
|
||||||
|
{% block header %}
|
||||||
|
{% endblock %}
|
||||||
|
</header>
|
||||||
|
<div class="content">
|
||||||
|
{% block content %}
|
||||||
|
{% endblock %}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
10
bitter/templates/home.html
Normal file
10
bitter/templates/home.html
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
{% extends "base.html" %}
|
||||||
|
{% block header %}
|
||||||
|
<h1>Hi. I'm a <strong>researcher</strong>.</h1>
|
||||||
|
{% endblock %}
|
||||||
|
{% block content %}
|
||||||
|
<p>Getting data from Twitter is hard, due to the limits imposed by the Twitter API</p>
|
||||||
|
<p>By logging in with my app you help me get more data for my research.</p>
|
||||||
|
<p>I will not use your personal information in any way.</p>
|
||||||
|
<a href="login" class="button button-big">Log in!</a>
|
||||||
|
{% endblock %}
|
||||||
11
bitter/templates/limits.html
Normal file
11
bitter/templates/limits.html
Normal file
@ -0,0 +1,11 @@
|
|||||||
|
{% extends "base.html" %}
|
||||||
|
{% block content %}
|
||||||
|
{% if limits | length %}
|
||||||
|
<ul id="double">
|
||||||
|
{% for limit in limits %}
|
||||||
|
<li class="limits"> <span>{{ limit }} </span><span><pre class="jsonlimits">{{ limits[limit] }}</pre></span></li>
|
||||||
|
{% endfor %}
|
||||||
|
</ul>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{% endblock %}
|
||||||
8
bitter/templates/sad.html
Normal file
8
bitter/templates/sad.html
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
{% extends "base.html" %}
|
||||||
|
{% block header %}
|
||||||
|
<h1>You don't wanna authorize me? :(</h1>
|
||||||
|
{% endblock %}
|
||||||
|
{% block content %}
|
||||||
|
<p>It's ok, but come back if you change your mind...</p>
|
||||||
|
|
||||||
|
{% endblock %}
|
||||||
10
bitter/templates/thanks.html
Normal file
10
bitter/templates/thanks.html
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
{% extends "base.html" %}
|
||||||
|
|
||||||
|
{% block header %}
|
||||||
|
Thanks!
|
||||||
|
{% endblock %}
|
||||||
|
{% block content %}
|
||||||
|
<p>I'll use it wisely</p>
|
||||||
|
<a href="login" class="button button-big">Authorize another account!</a>
|
||||||
|
{% endblock %}
|
||||||
|
|
||||||
@ -5,12 +5,17 @@ import json
|
|||||||
import signal
|
import signal
|
||||||
import sys
|
import sys
|
||||||
import sqlalchemy
|
import sqlalchemy
|
||||||
|
import os
|
||||||
|
|
||||||
from itertools import islice
|
from itertools import islice
|
||||||
|
from contextlib import contextmanager
|
||||||
|
|
||||||
from twitter import TwitterHTTPError
|
from twitter import TwitterHTTPError
|
||||||
|
|
||||||
from bitter.models import Following, User, ExtractorEntry, make_session
|
from bitter.models import Following, User, ExtractorEntry, make_session
|
||||||
|
|
||||||
|
from bitter import config
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
@ -19,12 +24,58 @@ def signal_handler(signal, frame):
|
|||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
||||||
|
|
||||||
|
def get_credentials_path(credfile=None):
|
||||||
|
if not credfile:
|
||||||
|
if config.CREDENTIALS:
|
||||||
|
credfile = config.CREDENTIALS
|
||||||
|
else:
|
||||||
|
raise Exception('No valid credentials file')
|
||||||
|
return os.path.expanduser(credfile)
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def credentials_file(credfile, *args, **kwargs):
|
||||||
|
p = get_credentials_path(credfile)
|
||||||
|
with open(p, *args, **kwargs) as f:
|
||||||
|
yield f
|
||||||
|
|
||||||
|
def iter_credentials(credfile=None):
|
||||||
|
with credentials_file(credfile) as f:
|
||||||
|
for l in f:
|
||||||
|
yield json.loads(l.strip())
|
||||||
|
|
||||||
|
def get_credentials(credfile=None, inverse=False, **kwargs):
|
||||||
|
creds = []
|
||||||
|
for i in iter_credentials(credfile):
|
||||||
|
if all(map(lambda x: i[x[0]] == x[1], kwargs.items())):
|
||||||
|
creds.append(i)
|
||||||
|
return creds
|
||||||
|
|
||||||
|
def create_credentials(credfile=None):
|
||||||
|
credfile = get_credentials_path(credfile)
|
||||||
|
with credentials_file(credfile, 'a'):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def delete_credentials(credfile=None, **creds):
|
||||||
|
tokeep = get_credentials(credfile, inverse=True, **creds)
|
||||||
|
with credentials_file(credfile, 'w') as f:
|
||||||
|
for i in tokeep:
|
||||||
|
f.write(json.dumps(i))
|
||||||
|
f.write('\n')
|
||||||
|
|
||||||
|
def add_credentials(credfile=None, **creds):
|
||||||
|
exist = get_credentials(credfile, **creds)
|
||||||
|
if not exist:
|
||||||
|
with credentials_file(credfile, 'a') as f:
|
||||||
|
f.write(json.dumps(creds))
|
||||||
|
f.write('\n')
|
||||||
|
|
||||||
|
|
||||||
def get_users(wq, ulist, by_name=False, queue=None, max_users=100):
|
def get_users(wq, ulist, by_name=False, queue=None, max_users=100):
|
||||||
t = 'name' if by_name else 'uid'
|
t = 'name' if by_name else 'uid'
|
||||||
logger.debug('Getting users by {}: {}'.format(t, ulist))
|
logger.debug('Getting users by {}: {}'.format(t, ulist))
|
||||||
ilist = iter(ulist)
|
ilist = iter(ulist)
|
||||||
while True:
|
while True:
|
||||||
userslice = ",".join(islice(ilist, max_users))
|
userslice = ",".join(str(i) for i in islice(ilist, max_users))
|
||||||
if not userslice:
|
if not userslice:
|
||||||
break
|
break
|
||||||
try:
|
try:
|
||||||
@ -48,7 +99,7 @@ def get_users(wq, ulist, by_name=False, queue=None, max_users=100):
|
|||||||
|
|
||||||
def trim_user(user):
|
def trim_user(user):
|
||||||
if 'status' in user:
|
if 'status' in user:
|
||||||
del user['status']
|
del user['status']
|
||||||
if 'follow_request_sent' in user:
|
if 'follow_request_sent' in user:
|
||||||
del user['follow_request_sent']
|
del user['follow_request_sent']
|
||||||
if 'created_at' in user:
|
if 'created_at' in user:
|
||||||
@ -64,7 +115,7 @@ def add_user(session, user, enqueue=False):
|
|||||||
olduser = session.query(User).filter(User.id==user['id'])
|
olduser = session.query(User).filter(User.id==user['id'])
|
||||||
if olduser:
|
if olduser:
|
||||||
olduser.delete()
|
olduser.delete()
|
||||||
user = User(**user)
|
user = User(**user)
|
||||||
session.add(user)
|
session.add(user)
|
||||||
if extract:
|
if extract:
|
||||||
logging.debug('Adding entry')
|
logging.debug('Adding entry')
|
||||||
@ -76,12 +127,12 @@ def add_user(session, user, enqueue=False):
|
|||||||
entry.pending = True
|
entry.pending = True
|
||||||
entry.cursor = -1
|
entry.cursor = -1
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
|
|
||||||
# TODO: adapt to the crawler
|
# TODO: adapt to the crawler
|
||||||
def extract(wq, recursive=False, user=None, initfile=None, dburi=None, extractor_name=None):
|
def extract(wq, recursive=False, user=None, initfile=None, dburi=None, extractor_name=None):
|
||||||
signal.signal(signal.SIGINT, signal_handler)
|
signal.signal(signal.SIGINT, signal_handler)
|
||||||
|
|
||||||
w = wq.next()
|
w = wq.next()
|
||||||
if not dburi:
|
if not dburi:
|
||||||
dburi = 'sqlite:///%s.db' % extractor_name
|
dburi = 'sqlite:///%s.db' % extractor_name
|
||||||
@ -99,7 +150,7 @@ def extract(wq, recursive=False, user=None, initfile=None, dburi=None, extractor
|
|||||||
except ValueError:
|
except ValueError:
|
||||||
logger.info("Added screen_name")
|
logger.info("Added screen_name")
|
||||||
screen_names.append(user.split('@')[-1])
|
screen_names.append(user.split('@')[-1])
|
||||||
|
|
||||||
if user:
|
if user:
|
||||||
classify_user(user)
|
classify_user(user)
|
||||||
|
|
||||||
@ -123,7 +174,7 @@ def extract(wq, recursive=False, user=None, initfile=None, dburi=None, extractor
|
|||||||
total_users = session.query(sqlalchemy.func.count(User.id)).scalar()
|
total_users = session.query(sqlalchemy.func.count(User.id)).scalar()
|
||||||
logging.info('Total users: {}'.format(total_users))
|
logging.info('Total users: {}'.format(total_users))
|
||||||
def pending_entries():
|
def pending_entries():
|
||||||
pending = session.query(ExtractorEntry).filter(ExtractorEntry.pending == True).count()
|
pending = session.query(ExtractorEntry).filter(ExtractorEntry.pending == True).count()
|
||||||
logging.info('Pending: {}'.format(pending))
|
logging.info('Pending: {}'.format(pending))
|
||||||
return pending
|
return pending
|
||||||
|
|
||||||
@ -192,7 +243,7 @@ def extract(wq, recursive=False, user=None, initfile=None, dburi=None, extractor
|
|||||||
|
|
||||||
session.add(candidate)
|
session.add(candidate)
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
96
bitter/webserver.py
Normal file
96
bitter/webserver.py
Normal file
@ -0,0 +1,96 @@
|
|||||||
|
import json
|
||||||
|
import os.path
|
||||||
|
from flask import Flask, redirect, session, url_for, request, flash, render_template
|
||||||
|
from twitter import Twitter
|
||||||
|
from twitter import OAuth as TOAuth
|
||||||
|
|
||||||
|
from flask_oauthlib.client import OAuth
|
||||||
|
|
||||||
|
from . import utils
|
||||||
|
from . import config
|
||||||
|
|
||||||
|
oauth = OAuth()
|
||||||
|
twitter = oauth.remote_app('twitter',
|
||||||
|
base_url='https://api.twitter.com/1/',
|
||||||
|
request_token_url='https://api.twitter.com/oauth/request_token',
|
||||||
|
access_token_url='https://api.twitter.com/oauth/access_token',
|
||||||
|
authorize_url='https://api.twitter.com/oauth/authenticate',
|
||||||
|
consumer_key=config.CONSUMER_KEY,
|
||||||
|
consumer_secret=config.CONSUMER_SECRET
|
||||||
|
)
|
||||||
|
|
||||||
|
app = Flask(__name__)
|
||||||
|
|
||||||
|
@twitter.tokengetter
|
||||||
|
def get_twitter_token(token=None):
|
||||||
|
return session.get('twitter_token')
|
||||||
|
|
||||||
|
|
||||||
|
@app.route('/login')
|
||||||
|
def login():
|
||||||
|
if 'twitter_token' in session:
|
||||||
|
del session['twitter_token']
|
||||||
|
return twitter.authorize(callback='/oauth-authorized')
|
||||||
|
# next=request.args.get('next') or request.referrer or None))
|
||||||
|
|
||||||
|
@app.route('/oauth-authorized')
|
||||||
|
def oauth_authorized():
|
||||||
|
resp = twitter.authorized_response()
|
||||||
|
if resp is None:
|
||||||
|
flash(u'You denied the request to sign in.')
|
||||||
|
return redirect('/sad')
|
||||||
|
token = (
|
||||||
|
resp['oauth_token'],
|
||||||
|
resp['oauth_token_secret']
|
||||||
|
)
|
||||||
|
user = resp['screen_name']
|
||||||
|
session['twitter_token'] = token
|
||||||
|
session['twitter_user'] = user
|
||||||
|
new_creds = {"token_key": token[0],
|
||||||
|
"token_secret": token[1]}
|
||||||
|
|
||||||
|
utils.delete_credentials(user=user)
|
||||||
|
utils.add_credentials(user=user,
|
||||||
|
token_key=token[0],
|
||||||
|
token_secret=token[1],
|
||||||
|
consumer_key=config.CONSUMER_KEY,
|
||||||
|
consumer_secret=config.CONSUMER_SECRET)
|
||||||
|
flash('You were signed in as %s' % resp['screen_name'])
|
||||||
|
return redirect('/thanks')
|
||||||
|
|
||||||
|
@app.route('/thanks')
|
||||||
|
def thanks():
|
||||||
|
return render_template("thanks.html")
|
||||||
|
|
||||||
|
@app.route('/sad')
|
||||||
|
def sad():
|
||||||
|
return render_template("sad.html")
|
||||||
|
|
||||||
|
@app.route('/')
|
||||||
|
def index():
|
||||||
|
# return 'Please <a href="./login">LOG IN</a> to help with my research :)'
|
||||||
|
return render_template('home.html')
|
||||||
|
|
||||||
|
@app.route('/hall')
|
||||||
|
def hall():
|
||||||
|
names = [c['user'] for c in utils.get_credentials()]
|
||||||
|
return render_template('thanks.html', names=names)
|
||||||
|
|
||||||
|
@app.route('/limits')
|
||||||
|
def limits():
|
||||||
|
creds = utils.get_credentials()
|
||||||
|
limits = {}
|
||||||
|
for c in creds:
|
||||||
|
auth = TOAuth(c['token_key'],
|
||||||
|
c['token_secret'],
|
||||||
|
c['consumer_key'],
|
||||||
|
c['consumer_secret'])
|
||||||
|
t = Twitter(auth=auth)
|
||||||
|
limits[c["user"]] = json.dumps(t.application.rate_limit_status(), indent=2)
|
||||||
|
|
||||||
|
return render_template('limits.html', limits=limits)
|
||||||
|
|
||||||
|
app.secret_key = os.environ.get('SESSION_KEY', 'bitter is cool!')
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
app.run()
|
||||||
@ -1,3 +1,4 @@
|
|||||||
sqlalchemy
|
sqlalchemy
|
||||||
twitter
|
twitter
|
||||||
click
|
click
|
||||||
|
six
|
||||||
|
|||||||
2
setup.py
2
setup.py
@ -27,7 +27,7 @@ setup(
|
|||||||
author='J. Fernando Sanchez',
|
author='J. Fernando Sanchez',
|
||||||
author_email='balkian@gmail.com',
|
author_email='balkian@gmail.com',
|
||||||
url="http://balkian.com",
|
url="http://balkian.com",
|
||||||
version="0.3",
|
version="0.4",
|
||||||
install_requires=install_reqs,
|
install_requires=install_reqs,
|
||||||
tests_require=test_reqs,
|
tests_require=test_reqs,
|
||||||
include_package_data=True,
|
include_package_data=True,
|
||||||
|
|||||||
@ -1,4 +1,4 @@
|
|||||||
from unittests import TestCase
|
from unittest import TestCase
|
||||||
|
|
||||||
class TestModels(TestCase):
|
class TestModels(TestCase):
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user