diff --git a/bitter/__init__.py b/bitter/__init__.py index 3ac18db..f1552bd 100644 --- a/bitter/__init__.py +++ b/bitter/__init__.py @@ -3,5 +3,8 @@ Bitter module. A library and cli for Twitter using python-twitter. http://github.com/balkian/bitter """ -__version__ = '0.4.1' +from future.standard_library import install_aliases +install_aliases() + +__version__ = '0.5.0' __all__ = ['cli', 'config', 'crawlers', 'models', 'utils' ] diff --git a/bitter/cli.py b/bitter/cli.py index ad3733d..dbd5fac 100644 --- a/bitter/cli.py +++ b/bitter/cli.py @@ -7,7 +7,6 @@ import sqlalchemy.types import threading import sqlite3 -from six.moves import map, filter, queue from sqlalchemy import exists from bitter import utils, models, crawlers diff --git a/bitter/utils.py b/bitter/utils.py index a865e3e..f762558 100644 --- a/bitter/utils.py +++ b/bitter/utils.py @@ -6,9 +6,12 @@ import signal import sys import sqlalchemy import os +import multiprocessing +from multiprocessing.pool import ThreadPool from itertools import islice from contextlib import contextmanager +from itertools import zip_longest from twitter import TwitterHTTPError @@ -23,6 +26,16 @@ def signal_handler(signal, frame): logger.info('You pressed Ctrl+C!') sys.exit(0) +def chunk(iterable, n, fillvalue=None): + args = [iter(iterable)] * n + return zip_longest(*args, fillvalue=fillvalue) + +def parallel(func, source, chunksize=0, numcpus=multiprocessing.cpu_count()): + if chunksize: + source = chunk(source, chunksize) + p = ThreadPool(numcpus) + for i in p.imap(func, source): + yield i def get_credentials_path(credfile=None): if not credfile: diff --git a/requirements.txt b/requirements.txt index fa6c088..3096c17 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ sqlalchemy twitter click -six +future diff --git a/tests/test_utils.py b/tests/test_utils.py index c4c0f85..7e71b73 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,7 @@ from unittest import TestCase import os +import types from bitter import utils from bitter import config as c @@ -45,6 +46,16 @@ class TestUtils(TestCase): utils.delete_credentials(user="test") print(utils.get_credentials()) assert not utils.get_credentials(user="test") - - + + def test_parallel(self): + import time + def echo(i): + time.sleep(2) + return i + tic = time.time() + resp = utils.parallel(echo, [1,2,3]) + assert isinstance(resp, types.GeneratorType) + assert list(resp) == [1,2,3] + toc = time.time() + assert (tic-toc) < 6000