Remove makefiles from auto push/pull

Replace algorithm list with a tuple

Closes #56

See merge request senpy/senpy!25

.gitlab-ci.yml

@@ -31,29 +31,19 @@ test-2.7:
   variables:
     PYTHON_VERSION: "2.7"
 
-.image: &image_definition
+push:
   stage: push
   script:
-    - make -e push-$PYTHON_VERSION
+    - make -e push
   only:
     - tags
     - triggers
     - fix-makefiles
 
-push-3.5:
-  <<: *image_definition
-  variables:
-    PYTHON_VERSION: "3.5"
-
-push-2.7:
-  <<: *image_definition
-  variables:
-    PYTHON_VERSION: "2.7"
-
 push-latest:
-  <<: *image_definition
-  variables:
-    PYTHON_VERSION: latest
+  stage: push
+  script:
+    - make -e push-latest
   only:
     - master
     - triggers

.makefiles/git.mk

@@ -22,7 +22,4 @@ else
 	rm $(KEY_FILE)
 endif
 
-push:: git-push
-pull:: git-pull
-
-.PHONY:: commit tag push git-push git-pull push-github 
+.PHONY:: commit tag git-push git-pull push-github 

.makefiles/makefiles.mk

@@ -1,17 +1,15 @@
 makefiles-remote:
-	@git remote add makefiles ssh://git@lab.cluster.gsi.dit.upm.es:2200/docs/templates/makefiles.git 2>/dev/null || true
+	git ls-remote --exit-code  makefiles 2> /dev/null || git remote add makefiles ssh://git@lab.cluster.gsi.dit.upm.es:2200/docs/templates/makefiles.git
 
 makefiles-commit: makefiles-remote
 	git add -f .makefiles
 	git commit -em "Updated makefiles from ${NAME}"
 
 makefiles-push:
+	git fetch makefiles $(NAME)
 	git subtree push --prefix=.makefiles/ makefiles $(NAME)
 
 makefiles-pull: makefiles-remote
 	git subtree pull --prefix=.makefiles/ makefiles master --squash
 
-pull:: makefiles-pull
-push:: makefiles-push
-
-.PHONY:: makefiles-remote makefiles-commit makefiles-push makefiles-pull pull push
+.PHONY:: makefiles-remote makefiles-commit makefiles-push makefiles-pull

example-plugins/async_plugin.py

@@ -11,7 +11,7 @@ class Async(AnalysisPlugin):
     '''An example of an asynchronous module'''
     author = '@balkian'
     version = '0.2'
-    async = True
+    sync = False
 
     def _do_async(self, num_processes):
         pool = multiprocessing.Pool(processes=num_processes)

requirements.txt

@@ -12,3 +12,4 @@ rdflib-jsonld
 numpy
 scipy
 scikit-learn
+responses

senpy/api.py

@@ -175,8 +175,8 @@ def parse_params(indict, *specs):
             parameters=outdict,
             errors=wrong_params)
         raise message
-    if 'algorithm' in outdict and not isinstance(outdict['algorithm'], list):
-        outdict['algorithm'] = list(outdict['algorithm'].split(','))
+    if 'algorithm' in outdict and not isinstance(outdict['algorithm'], tuple):
+        outdict['algorithm'] = tuple(outdict['algorithm'].split(','))
     return outdict
 
 

senpy/blueprints.py

@@ -69,7 +69,7 @@ def encoded_url(url=None, base=None):
         if request.method == 'GET':
             url = request.full_path[1:]  # Remove the first slash
         else:
-            hash(frozenset(request.form.params().items()))
+            hash(frozenset(tuple(request.parameters.items())))
             code = 'hash:{}'.format(hash)
 
     code = code or base64.urlsafe_b64encode(url.encode()).decode()
@@ -184,14 +184,19 @@ def basic_api(f):
     return decorated_function
 
 
-@api_blueprint.route('/', methods=['POST', 'GET'])
+@api_blueprint.route('/', defaults={'plugin': None}, methods=['POST', 'GET'])
+@api_blueprint.route('/<path:plugin>', methods=['POST', 'GET'])
 @basic_api
-def api_root():
+def api_root(plugin):
     if request.parameters['help']:
         dic = dict(api.API_PARAMS, **api.NIF_PARAMS)
         response = Help(valid_parameters=dic)
         return response
     req = api.parse_call(request.parameters)
+    if plugin:
+        plugin = plugin.replace('+', '/')
+        plugin = plugin.split('/')
+        req.parameters['algorithm'] = tuple(plugin)
     return current_app.senpy.analyse(req)
 
 
@@ -221,7 +226,7 @@ def plugins():
 
 @api_blueprint.route('/plugins/<plugin>/', methods=['POST', 'GET'])
 @basic_api
-def plugin(plugin=None):
+def plugin(plugin):
     sp = current_app.senpy
     return sp.get_plugin(plugin)
 

senpy/client.py

@@ -13,7 +13,7 @@ class Client(object):
         return self.request('/', method=method, input=input, **kwargs)
 
     def evaluate(self, input, method='GET', **kwargs):
-        return self.request('/evaluate', method = method, input=input, **kwargs)
+        return self.request('/evaluate', method=method, input=input, **kwargs)
 
     def plugins(self, *args, **kwargs):
         resp = self.request(path='/plugins').plugins
@@ -24,8 +24,12 @@ class Client(object):
         return {d.name: d for d in resp}
 
     def request(self, path=None, method='GET', **params):
-        url = '{}{}'.format(self.endpoint, path)
-        response = requests.request(method=method, url=url, params=params)
+        url = '{}{}'.format(self.endpoint.rstrip('/'), path)
+        if method == 'POST':
+            response = requests.post(url=url, data=params)
+        else:
+            response = requests.request(method=method, url=url, params=params)
+
         try:
             resp = models.from_dict(response.json())
         except Exception as ex:

senpy/extensions.py

@@ -153,7 +153,6 @@ class Senpy(object):
                 yield i
             return
         plugin = plugins[0]
-        self._activate(plugin)  # Make sure the plugin is activated
         specific_params = api.parse_extra_params(req, plugin)
         req.analysis.append({'plugin': plugin,
                              'parameters': specific_params})
@@ -352,7 +351,7 @@ class Senpy(object):
 
         logger.info("Activating plugin: {}".format(plugin.name))
 
-        if sync or 'async' in plugin and not plugin.async:
+        if sync or not getattr(plugin, 'async', True) or getattr(plugin, 'sync', False):
             return self._activate(plugin)
         else:
             th = Thread(target=partial(self._activate, plugin))
@@ -375,7 +374,7 @@ class Senpy(object):
 
         self._set_active(plugin, False)
 
-        if sync or 'async' in plugin and not plugin.async:
+        if sync or not getattr(plugin, 'async', True) or not getattr(plugin, 'sync', False):
             self._deactivate(plugin)
         else:
             th = Thread(target=partial(self._deactivate, plugin))

senpy/gsitk_compat.py

@@ -1,5 +1,7 @@
 import logging
 
+from pkg_resources import parse_version, get_distribution, DistributionNotFound
+
 logger = logging.getLogger(__name__)
 
 MSG = 'GSITK is not (properly) installed.'
@@ -12,12 +14,18 @@ def raise_exception(*args, **kwargs):
 
 
 try:
+    gsitk_distro = get_distribution("gsitk")
+    GSITK_VERSION = parse_version(gsitk_distro.version)
+    GSITK_AVAILABLE = GSITK_VERSION > parse_version("0.1.9.1")  # Earlier versions have a bug
+except DistributionNotFound:
+    GSITK_AVAILABLE = False
+    GSITK_VERSION = ()
+
+if GSITK_AVAILABLE:
     from gsitk.datasets.datasets import DatasetManager
     from gsitk.evaluation.evaluation import Evaluation as Eval
     from sklearn.pipeline import Pipeline
-    GSITK_AVAILABLE = True
     modules = locals()
-except ImportError:
-    logger.warn(IMPORTMSG)
-    GSITK_AVAILABLE = False
+else:
+    logger.warning(IMPORTMSG)
     DatasetManager = Eval = Pipeline = raise_exception

senpy/plugins/__init__.py

@@ -24,6 +24,7 @@ import nltk
 from .. import models, utils
 from .. import api
 from .. import gsitk_compat
+from .. import testing
 
 
 logger = logging.getLogger(__name__)
@@ -143,14 +144,23 @@ class Plugin(with_metaclass(PluginMeta, models.Plugin)):
                 self.log.warn('Test case failed:\n{}'.format(pprint.pformat(case)))
                 raise
 
-    def test_case(self, case):
+    def test_case(self, case, mock=testing.MOCK_REQUESTS):
         entry = models.Entry(case['entry'])
         given_parameters = case.get('params', case.get('parameters', {}))
         expected = case.get('expected', None)
         should_fail = case.get('should_fail', False)
+        responses = case.get('responses', [])
+
         try:
             params = api.parse_params(given_parameters, self.extra_params)
-            res = list(self.analyse_entries([entry, ], params))
+
+            method = partial(self.analyse_entries, [entry, ], params)
+
+            if mock:
+                res = list(method())
+            else:
+                with testing.patch_all_requests(responses):
+                    res = list(method())
 
             if not isinstance(expected, list):
                 expected = [expected]

senpy/plugins/sentiment/sentiment140/sentiment140_plugin.py

@@ -4,6 +4,8 @@ import json
 from senpy.plugins import SentimentPlugin
 from senpy.models import Sentiment
 
+ENDPOINT = 'http://www.sentiment140.com/api/bulkClassifyJson'
+
 
 class Sentiment140(SentimentPlugin):
     '''Connects to the sentiment140 free API: http://sentiment140.com'''
@@ -26,7 +28,7 @@ class Sentiment140(SentimentPlugin):
 
     def analyse_entry(self, entry, params):
         lang = params["language"]
-        res = requests.post("http://www.sentiment140.com/api/bulkClassifyJson",
+        res = requests.post(ENDPOINT,
                             json.dumps({
                                 "language": lang,
                                 "data": [{
@@ -52,18 +54,6 @@ class Sentiment140(SentimentPlugin):
         entry.language = lang
         yield entry
 
-    def test(self, *args, **kwargs):
-        '''
-        To avoid calling the sentiment140 API, we will mock the results
-        from requests.
-        '''
-        from senpy.testing import patch_requests
-        expected = {"data": [{"polarity": 4}]}
-        with patch_requests(expected) as (request, response):
-            super(Sentiment140, self).test(*args, **kwargs)
-            assert request.called
-            assert response.json.called
-
     test_cases = [
         {
             'entry': {
@@ -77,6 +67,9 @@ class Sentiment140(SentimentPlugin):
                         'marl:hasPolarity': 'marl:Positive',
                     }
                 ]
-            }
+            },
+            'responses': [{'url': ENDPOINT,
+                           'method': 'POST',
+                           'json': {'data': [{'polarity': 4}]}}]
         }
     ]

senpy/testing.py

@@ -1,36 +1,31 @@
-try:
-    from unittest.mock import patch, MagicMock
-except ImportError:
-    from mock import patch, MagicMock
-
 from past.builtins import basestring
 
-
-import json
-from contextlib import contextmanager
+import os
+import responses as requestmock
 
 from .models import BaseModel
 
 
-@contextmanager
-def patch_requests(value, code=200):
-    success = MagicMock()
-    if isinstance(value, BaseModel):
-        value = value.jsonld()
-    if not isinstance(value, basestring):
-        data = json.dumps(value)
+MOCK_REQUESTS = os.environ.get('MOCK_REQUESTS', '').lower() in ['no', 'false']
+
+
+def patch_all_requests(responses):
+
+    patched = requestmock.RequestsMock()
+
+    for response in responses or []:
+        args = response.copy()
+        if 'json' in args and isinstance(args['json'], BaseModel):
+            args['json'] = args['json'].jsonld()
+        args['method'] = getattr(requestmock, args.get('method', 'GET'))
+        patched.add(**args)
+    return patched
+
+
+def patch_requests(url, response, method='GET', status=200):
+    args = {'url': url, 'method': method, 'status': status}
+    if isinstance(response, basestring):
+        args['body'] = response
     else:
-        data = value
-
-    success.json.return_value = value
-
-    success.status_code = code
-    success.content = data
-    success.text = data
-
-    method_mocker = MagicMock()
-    method_mocker.return_value = success
-    with patch.multiple('requests', request=method_mocker,
-                        get=method_mocker, post=method_mocker):
-        yield method_mocker, success
-        assert method_mocker.called
+        args['json'] = response
+    return patch_all_requests([args])

tests/test_blueprints.py

@@ -67,10 +67,25 @@ class BlueprintsTest(TestCase):
         logging.debug("Got response: %s", js)
         assert "@context" in js
         assert "entries" in js
+        assert len(js['analysis']) == 1
+
+    def test_analysis_post(self):
+        """
+        The results for a POST request should be the same as for a GET request.
+        """
+        resp = self.client.post("/api/", data={'i': 'My aloha mohame',
+                                               'algorithm': 'rand',
+                                               'with_parameters': True})
+        self.assertCode(resp, 200)
+        js = parse_resp(resp)
+        logging.debug("Got response: %s", js)
+        assert "@context" in js
+        assert "entries" in js
+        assert len(js['analysis']) == 1
 
     def test_analysis_extra(self):
         """
-        Extra params that have a default should
+        Extra params that have a default should use it
         """
         resp = self.client.get("/api/?i=My aloha mohame&algo=Dummy&with_parameters=true")
         self.assertCode(resp, 200)
@@ -95,6 +110,44 @@ class BlueprintsTest(TestCase):
         resp = self.client.get("/api/?i=My aloha mohame&algo=DummyRequired&example=a")
         self.assertCode(resp, 200)
 
+    def test_analysis_url(self):
+        """
+        The algorithm can also be specified as part of the URL
+        """
+        self.app.config['TESTING'] = False  # Errors are expected in this case
+        resp = self.client.get("/api/DummyRequired?i=My aloha mohame")
+        self.assertCode(resp, 400)
+        js = parse_resp(resp)
+        logging.debug("Got response: %s", js)
+        assert isinstance(js, models.Error)
+        resp = self.client.get("/api/DummyRequired?i=My aloha mohame&example=notvalid")
+        self.assertCode(resp, 400)
+        resp = self.client.get("/api/DummyRequired?i=My aloha mohame&example=a")
+        self.assertCode(resp, 200)
+
+    def test_analysis_chain(self):
+        """
+        More than one algorithm can be specified. Plugins will then be chained
+        """
+        resp = self.client.get("/api/Dummy?i=My aloha mohame")
+        js = parse_resp(resp)
+        assert len(js['analysis']) == 1
+        assert js['entries'][0]['nif:isString'] == 'My aloha mohame'[::-1]
+
+        resp = self.client.get("/api/Dummy/Dummy?i=My aloha mohame")
+        # Calling dummy twice, should return the same string
+        self.assertCode(resp, 200)
+        js = parse_resp(resp)
+        assert len(js['analysis']) == 2
+        assert js['entries'][0]['nif:isString'] == 'My aloha mohame'
+
+        resp = self.client.get("/api/Dummy+Dummy?i=My aloha mohame")
+        # Same with pluses instead of slashes
+        self.assertCode(resp, 200)
+        js = parse_resp(resp)
+        assert len(js['analysis']) == 2
+        assert js['entries'][0]['nif:isString'] == 'My aloha mohame'
+
     def test_error(self):
         """
         The dummy plugin returns an empty response,\

tests/test_client.py

@@ -14,22 +14,28 @@ class ModelsTest(TestCase):
     def test_client(self):
         endpoint = 'http://dummy/'
         client = Client(endpoint)
-        with patch_requests(Results()) as (request, response):
+        with patch_requests('http://dummy/', Results()):
             resp = client.analyse('hello')
             assert isinstance(resp, Results)
-        request.assert_called_with(
-            url=endpoint + '/', method='GET', params={'input': 'hello'})
-        with patch_requests(Error('Nothing')) as (request, response):
+        with patch_requests('http://dummy/', Error('Nothing')):
             try:
                 client.analyse(input='hello', algorithm='NONEXISTENT')
                 raise Exception('Exceptions should be raised. This is not golang')
             except Error:
                 pass
-        request.assert_called_with(
-            url=endpoint + '/',
-            method='GET',
-            params={'input': 'hello',
-                    'algorithm': 'NONEXISTENT'})
+
+    def test_client_post(self):
+        endpoint = 'http://dummy/'
+        client = Client(endpoint)
+        with patch_requests('http://dummy/', Results()):
+            resp = client.analyse('hello')
+            assert isinstance(resp, Results)
+        with patch_requests('http://dummy/', Error('Nothing'), method='POST'):
+            try:
+                client.analyse(input='hello', method='POST', algorithm='NONEXISTENT')
+                raise Exception('Exceptions should be raised. This is not golang')
+            except Error:
+                pass
 
     def test_plugins(self):
         endpoint = 'http://dummy/'
@@ -37,11 +43,8 @@ class ModelsTest(TestCase):
         plugins = Plugins()
         p1 = AnalysisPlugin({'name': 'AnalysisP1', 'version': 0, 'description': 'No'})
         plugins.plugins = [p1, ]
-        with patch_requests(plugins) as (request, response):
+        with patch_requests('http://dummy/plugins', plugins):
             response = client.plugins()
             assert isinstance(response, dict)
             assert len(response) == 1
             assert "AnalysisP1" in response
-        request.assert_called_with(
-            url=endpoint + '/plugins', method='GET',
-            params={})

tests/test_plugins.py

@@ -1,15 +1,15 @@
 #!/bin/env python
 
 import os
-import sys
 import pickle
 import shutil
 import tempfile
 
-from unittest import TestCase
+from unittest import TestCase, skipIf
 from senpy.models import Results, Entry, EmotionSet, Emotion, Plugins
 from senpy import plugins
 from senpy.plugins.conversion.emotion.centroids import CentroidConversion
+from senpy.gsitk_compat import GSITK_AVAILABLE
 
 import pandas as pd
 
@@ -346,13 +346,15 @@ class PluginsTest(TestCase):
         smart_metrics = results[0].metrics[0]
         assert abs(smart_metrics['accuracy'] - 1) < 0.01
 
+    @skipIf(not GSITK_AVAILABLE, "GSITK is not available")
     def test_evaluation(self):
-        if sys.version_info < (3, 0):
-            with self.assertRaises(Exception) as context:
-                self._test_evaluation()
-            self.assertTrue('GSITK ' in str(context.exception))
-        else:
+        self._test_evaluation()
+
+    @skipIf(GSITK_AVAILABLE, "GSITK is available")
+    def test_evaluation_unavailable(self):
+        with self.assertRaises(Exception) as context:
             self._test_evaluation()
+        self.assertTrue('GSITK ' in str(context.exception))
 
 
 def make_mini_test(fpath):

tests/test_test.py

@@ -5,28 +5,29 @@ import json
 from senpy.testing import patch_requests
 from senpy.models import Results
 
+ENDPOINT = 'http://example.com'
+
 
 class TestTest(TestCase):
     def test_patch_text(self):
-        with patch_requests('hello'):
-            r = requests.get('http://example.com')
+        with patch_requests(ENDPOINT, 'hello'):
+            r = requests.get(ENDPOINT)
             assert r.text == 'hello'
-            assert r.content == 'hello'
 
     def test_patch_json(self):
         r = Results()
-        with patch_requests(r):
-            res = requests.get('http://example.com')
-            assert res.content == json.dumps(r.jsonld())
+        with patch_requests(ENDPOINT, r):
+            res = requests.get(ENDPOINT)
+            assert res.text == json.dumps(r.jsonld())
             js = res.json()
             assert js
             assert js['@type'] == r['@type']
 
     def test_patch_dict(self):
         r = {'nothing': 'new'}
-        with patch_requests(r):
-            res = requests.get('http://example.com')
-            assert res.content == json.dumps(r)
+        with patch_requests(ENDPOINT, r):
+            res = requests.get(ENDPOINT)
+            assert res.text == json.dumps(r)
             js = res.json()
             assert js
             assert js['nothing'] == 'new'