Parallel processing and future

bitter/__init__.py

@@ -3,5 +3,8 @@ Bitter module. A library and cli for Twitter using python-twitter.
 http://github.com/balkian/bitter
 """
 
-__version__ = '0.4.1'
+from future.standard_library import install_aliases
+install_aliases()
+
+__version__ = '0.5.0'
 __all__ = ['cli', 'config', 'crawlers', 'models', 'utils' ]

bitter/cli.py

@@ -7,7 +7,6 @@ import sqlalchemy.types
 import threading
 import sqlite3
 
-from six.moves import map, filter, queue
 from sqlalchemy import exists
 
 from bitter import utils, models, crawlers

bitter/utils.py

@@ -6,9 +6,12 @@ import signal
 import sys
 import sqlalchemy
 import os
+import multiprocessing
+from multiprocessing.pool import ThreadPool
 
 from itertools import islice
 from contextlib import contextmanager
+from itertools import zip_longest
 
 from twitter import TwitterHTTPError
 
@@ -23,6 +26,16 @@ def signal_handler(signal, frame):
     logger.info('You pressed Ctrl+C!')
     sys.exit(0)
 
+def chunk(iterable, n, fillvalue=None):
+    args = [iter(iterable)] * n
+    return zip_longest(*args, fillvalue=fillvalue)
+
+def parallel(func, source, chunksize=0, numcpus=multiprocessing.cpu_count()):
+    if chunksize:
+        source = chunk(source, chunksize)
+    p = ThreadPool(numcpus)
+    for i in p.imap(func, source):
+        yield i
 
 def get_credentials_path(credfile=None):
     if not credfile:

requirements.txt

@@ -1,4 +1,4 @@
 sqlalchemy
 twitter
 click
-six
+future

tests/test_utils.py

@@ -1,6 +1,7 @@
 from unittest import TestCase
 
 import os
+import types
 
 from bitter import utils
 from bitter import config as c
@@ -45,6 +46,16 @@ class TestUtils(TestCase):
         utils.delete_credentials(user="test")
         print(utils.get_credentials())
         assert not utils.get_credentials(user="test")
-        
+
-        
+    def test_parallel(self):
+        import time
+        def echo(i):
+            time.sleep(2)
+            return i
+        tic = time.time()
+        resp = utils.parallel(echo, [1,2,3])
+        assert isinstance(resp, types.GeneratorType)
+        assert list(resp) == [1,2,3]
+        toc = time.time()
+        assert (tic-toc) < 6000