commit 96134709bb1be6303b04f84768371183a533c8fd Author: J. Fernando Sánchez Date: Mon Oct 18 09:30:50 2021 +0200 First version diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..42245a3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ + +.* +*.pyc +__pycache__ +dist +build \ No newline at end of file diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..e4fba20 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +include requirements.txt +include test-requirements.txt +include extra-requirements.txt +include README.md +include LICENSE.txt +graft keepit diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1206496 --- /dev/null +++ b/Makefile @@ -0,0 +1,4 @@ +release: + python setup.py sdist + python setup.py bdist_wheel + twine upload --skip-existing dist/* diff --git a/README.md b/README.md new file mode 100644 index 0000000..63ae93b --- /dev/null +++ b/README.md @@ -0,0 +1,41 @@ +# KEEP IT + +This is a **WORK IN PROGRESS**. + +`keepit` provides advanced memoization to disk for functions. +In other words, it records the results of important functions between executions. + +`keepit` saves the results of calling a function to disk, so calling the function with the exact same parameters will re-use the stored copy of the results, leading to much faster times. + + +Example usage: + +``` +import pandas as pd +from keepit import keepit + +@keepit('myresults.tsv') +def expensive_function(number=1): + df = pd.DataFrame() + # Perform a really expensive operation, maybe access to disk? + return df + +# When a results file for the function does not exist +# this may take a long time +expensive_function(number=1) +# Now a myresults.tsv_{some hash) has been generated + +# This is almost instantaneous: +expensive_function(number=1) + +# Files are specific to each parameter execution, +# so this will again take a long time: +expensive_function(number=42) +# After this, we should have two files, one for number=1, +# and another one for number=42. + +``` + + + + diff --git a/extra-requirements.txt b/extra-requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/keepit/VERSION b/keepit/VERSION new file mode 100644 index 0000000..f477849 --- /dev/null +++ b/keepit/VERSION @@ -0,0 +1 @@ +0.2.2 \ No newline at end of file diff --git a/keepit/__init__.py b/keepit/__init__.py new file mode 100644 index 0000000..a5183b1 --- /dev/null +++ b/keepit/__init__.py @@ -0,0 +1,130 @@ +import os +import unicodedata +import inspect +import re +import pandas as pd +from functools import wraps, partial +from glob import glob +import logging +import pickle +import hashlib + +from collections import namedtuple + +from .backends import Pickle, Entry + +logger = logging.getLogger(__name__) + + +def _slugify(value): + """ + Normalizes string, converts to lowercase, removes non-alpha characters, + and converts spaces to hyphens. + Source: + http://stackoverflow.com/questions/295135/turn-a-string-into-a-valid-filename-in-python + """ + value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore') + value = re.sub(r'[^\w\s-]', '', value.decode('utf-8', 'ignore')) + value = value.strip().lower() + value = re.sub(r'[-\s]+', '-', value) + return value + + +def hash_df(df): + '''Hashes a pandas dataframe''' + return hashlib.sha256(pd.util.hash_pandas_object(df, index=True).values).hexdigest() + + +def hash_object(obj): + return hashlib.sha256(pickle.dumps(obj)).hexdigest() + + +def hash_element(elem): + if isinstance(elem, pd.DataFrame): + return hash_df(elem) + pick = pickle.dumps(elem) + return hashlib.sha256(pick).hexdigest() + + +def func_hasher(f, *args, fname=None, element_hasher=hash_element, **kwargs): + + sig = inspect.signature(f) + func = partial(f, *args, **kwargs) + bound = sig.bind_partial(*args, **kwargs) + bound.apply_defaults() + + reqs = {} + for k, v in bound.arguments.items(): + reqs[k] = v + + fname = fname or '{}_{}'.format(f.__name__, f.__module__) + + args_hash = hash_object(reqs) + name = '{}_{}'.format(fname, args_hash) + + return _slugify(name), func, reqs + + +Result = namedtuple('Result', ['func', 'args', 'value']) + + +class HashedFunc: + + def __init__(self, func, fname=None, tags=[], backend=Pickle()): + self.func = func + self.tags = tags + self.backend = backend + self.sig = inspect.signature(func) + self.fname = fname or '{}_{}'.format(self.func.__name__, + self.func.__module__) + + def hash(self, *args, **kwargs): + return func_hasher(self.func, *args, fname=self.fname, **kwargs, element_hasher=self.hash_element) + + def hash_element(self, elem): + return hash_element(elem) + + def __call__(self, *args, cache_force=False, tags=[], **kwargs): + + if os.environ.get('no_cache'): + return self.func(*args, **kwargs) + + func_id, func, requirements = self.hash(*args, **kwargs) + print(func_id) + + if cache_force or not self.backend.exists(func_id): + res = func() + e = Entry(tags=[self.fname, ], + id=func_id, + content=Result(func_id, requirements, res)) + self.backend.put(e) + # hash = self.hash_element(res) + # self.backend.put(hash, res, tags=self.tags+tags) + # for req_hash, req_value in requirements.items(): + # if not self.backend.find(req_hash): + # self.backend.put(req_hash, req_value) + else: + res = self.backend.get(func_id).content.value + return res + + def drop(self, *args, **kwargs): + func_id, func, requirements = self.hash(*args, **kwargs) + if self.backend.exists(func_id): + self.backend.remove(func_id) + + def drop_all(self): + for f in self.list(): + self.backend.remove(f.id) + + def list(self): + return list(self.backend.find(tags=[self.fname, ])) + + +def keepit(fname=None, hasher=HashedFunc, **kwargs): + def outer(of): + return hasher(of, fname=fname, **kwargs) + return outer + + +def diff(df1, df2): + return pd.concat([df1, df2]).drop_duplicates(keep=False) diff --git a/keepit/backends/__init__.py b/keepit/backends/__init__.py new file mode 100644 index 0000000..e32bcc2 --- /dev/null +++ b/keepit/backends/__init__.py @@ -0,0 +1,97 @@ +import os +import pickle +import time +import sqlite3 + +from glob import glob + +from pathlib import Path + +ROOT = os.path.join(Path.home(), '.keepit') + + +ROOT = os.path.abspath(os.path.basename(__file__)) +CACHE_DIR = os.environ.get('CACHE_DIR', os.path.join(ROOT, '_cache')) + + +class NotFound(Exception): + pass + + +class BackEnd(): + + def exists(self, oid): + raise NotImplementedError() + + def read(self, oid): + raise NotImplementedError() + + def put(self, entry): + raise NotImplementedError() + + def remove(self, oid): + raise NotImplementedError() + + def find(self, *args, **kwargs): + return list(self.ifind(*args, **kwargs)) + + def ifind(self, oid=None, tags=[]): + raise NotImplementedError() + + def erase_all(self): + for entry in self.find(): + self.remove(entry.id) + + +class Entry: + + def __init__(self, id, content, tags=[], timestamp=None): + self.id = id + self.timestamp = time.localtime(timestamp or time.time()) + self.content = content + self.tags = set(tags) + + def __repr__(self): + return str(self) + + def __str__(self): + return '{} @ {} [{}]'.format(self.id, time.strftime('%Y-%m-%d %H:%M', self.timestamp), ','.join(self.tags)) + + +class Pickle(BackEnd): + res_folder = 'keepit_cache' + + def __init__(self): + pass + + def _filename(self, oid): + return os.path.join(self.res_folder, "{}.pickle".format(oid)) + + def _open(self, fpath, abs=False): + if not abs: + fpath = self._filename(fpath) + with open(fpath, 'rb') as f: + return pickle.load(f) + + def put(self, entry): + if not os.path.exists(self.res_folder): + os.makedirs(self.res_folder) + with open(self._filename(entry.id), 'wb') as f: + pickle.dump(entry, f) + + def exists(self, oid): + return os.path.exists(self._filename(oid)) + + def get(self, oid): + s = self._open(oid) + return s + + def remove(self, oid): + return os.remove(self._filename(oid)) + + def ifind(self, oid=None, tags=[]): + target = set(tags) + for f in glob(os.path.join(self.res_folder, '*')): + e = self._open(f, abs=True) + if (not oid or f.id == oid) and e.tags.issuperset(tags): + yield e diff --git a/keepit/version.py b/keepit/version.py new file mode 100644 index 0000000..c517050 --- /dev/null +++ b/keepit/version.py @@ -0,0 +1,19 @@ +import os +import logging + +logger = logging.getLogger(__name__) + +ROOT = os.path.dirname(__file__) +DEFAULT_FILE = os.path.join(ROOT, 'VERSION') + + +def read_version(versionfile=DEFAULT_FILE): + try: + with open(versionfile) as f: + return f.read().strip() + except IOError: # pragma: no cover + logger.error('Running an unknown version of senpy. Be careful!.') + return '0.0' + + +__version__ = read_version() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..1c8be29 --- /dev/null +++ b/setup.py @@ -0,0 +1,51 @@ +from setuptools import setup + +with open('keepit/VERSION') as f: + __version__ = f.read().strip() + assert __version__ + + +def parse_requirements(filename): + """ load requirements from a pip requirements file """ + with open(filename, 'r') as f: + lineiter = list(line.strip() for line in f) + return [line for line in lineiter if line and not line.startswith("#")] + + +install_reqs = parse_requirements("requirements.txt") +test_reqs = parse_requirements("test-requirements.txt") + + +# read the contents of your README file +from os import path +this_directory = path.abspath(path.dirname(__file__)) +with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f: + long_description = f.read() + + +setup( + name='keepit', + python_requires='>3.3', + packages=['keepit'], # this must be the same as the name above + version=__version__, + license='Apache License 2.0', + description=('advanced memoization/caching of functions with data analytics in mind'), + long_description=long_description, + long_description_content_type='text/markdown', + author='J. Fernando Sanchez', + author_email='balkian@gmail.com', + url='https://github.com/balkian/keepit', # use the URL to the github repo + download_url='https://github.com/balkian/keepit/archive/{}.tar.gz'.format( + __version__), + keywords=['data analysis', 'memoization', 'cache'], + classifiers=[ + 'Programming Language :: Python :: 3', + ], + install_requires=install_reqs, + tests_require=test_reqs, + setup_requires=['pytest-runner', ], + include_package_data=True, + entry_points={ + 'console_scripts': + ['keepit = keepit.__main__:main',] + }) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/tests/create_custom.py b/tests/create_custom.py new file mode 100644 index 0000000..c10b667 --- /dev/null +++ b/tests/create_custom.py @@ -0,0 +1,13 @@ +from keepit import keepit + +ORIGIN = None + + +@keepit('myfunction') +def custom_function(): + return ORIGIN + + +if __name__ == '__main__': + ORIGIN = 'main' + custom_function() diff --git a/tests/test.tsv b/tests/test.tsv new file mode 100644 index 0000000..bb15a53 --- /dev/null +++ b/tests/test.tsv @@ -0,0 +1,3 @@ + int str bool other +0 0 hola True +1 1 adiós False 5.0 diff --git a/tests/test_main.py b/tests/test_main.py new file mode 100644 index 0000000..9b54049 --- /dev/null +++ b/tests/test_main.py @@ -0,0 +1,98 @@ +import sys +import os +import subprocess +from unittest import TestCase + +from keepit import keepit, diff, hash_df +from keepit.backends import Pickle + +import pandas as pd + +# df = pd.DataFrame([[0, 'hola', True, None], [1, 'adiós', False, 5]], columns=['int', 'str', 'bool', 'other']) + +this_directory = os.path.abspath(os.path.dirname(__file__)) + + +count = 0 + +@keepit() +def prueba(first, name='something', value=29): + global count + count += 1 + return pd.DataFrame([[count, first, name, value], ], columns=['time', 'first', 'name', 'value']) + + +@keepit() +def prueba_df(): + return pd.read_csv(os.path.join(this_directory, 'test.tsv'), sep='\t') + + +@keepit() +def prueba_df_argument(df): + return count + + +class TestMain(TestCase): + + + def setUpClass(): + back = Pickle() + back.erase_all() + assert not back.find() + assert (not os.path.exists(back.res_folder)) or (not os.listdir(back.res_folder)) + + def tearDownClass(): + back = Pickle() + back.erase_all() + + def test_basic(self): + + try: + prueba() + except TypeError: + pass + + p1 = prueba('hello') + print(p1) + p2 = prueba('hello') + print(p2) + print(diff(p1, p2)) + assert p1.equals(p2) # Columns and rows are equal + + p3 = prueba('other') + print(p3) + assert not p1.equals(p3) + + p4 = prueba('hello', name='different') + print(p4) + assert not p1.equals(p4) + + def test_list_and_drop(self): + assert len(prueba.list()) > 0 + prueba.drop_all() + assert len(prueba.list()) == 0 + + def test_df(self): + df1 = prueba_df() + df2 = prueba_df() + + assert df1.equals(df2) + assert hash_df(df1) == hash_df(df2) + prueba_df.drop_all() + + def test_different_program(self): + '''A value saved by a different interpreter should be reusable.''' + from create_custom import custom_function + custom_function.drop_all() + subprocess.check_call([sys.executable, os.path.join(this_directory, 'create_custom.py')]) + # create_custom returns different results when run as a script. + # We make sure we are using the "stored" value, and not the result from the imported function + assert custom_function() == 'main' + + def test_df_arg(self): + + df1 = pd.read_csv(os.path.join(this_directory, 'test.tsv'), sep='\t') + res1 = prueba_df_argument(df1) + df2 = pd.read_csv(os.path.join(this_directory, 'test.tsv'), sep='\t') + res2 = prueba_df_argument(df2) + assert res1 == res2