mirror of https://github.com/balkian/bitter.git
You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
136 lines
4.2 KiB
Python
136 lines
4.2 KiB
Python
import time
|
|
import json
|
|
|
|
from sqlalchemy.ext.declarative import declarative_base
|
|
from sqlalchemy.types import BigInteger, Integer, Text, Boolean
|
|
from sqlalchemy.schema import ForeignKey
|
|
from sqlalchemy.pool import SingletonThreadPool
|
|
from sqlalchemy import Column, Index
|
|
|
|
from sqlalchemy import create_engine
|
|
from sqlalchemy.orm import sessionmaker
|
|
from functools import wraps
|
|
|
|
Base = declarative_base()
|
|
|
|
|
|
class User(Base):
|
|
__tablename__ = 'users'
|
|
|
|
id = Column(BigInteger, primary_key=True, index=True, unique=True)
|
|
contributors_enabled = Column(Boolean)
|
|
created_at_stamp = Column(Text)
|
|
default_profile = Column(Boolean)
|
|
default_profile_image = Column(Boolean)
|
|
description = Column(Text)
|
|
entities = Column(Text)
|
|
favourites_count = Column(Integer)
|
|
followers_count = Column(Integer)
|
|
following = Column(Boolean)
|
|
friends_count = Column(Integer)
|
|
geo_enabled = Column(Boolean)
|
|
has_extended_profile = Column(Boolean)
|
|
id_str = Column(Text)
|
|
is_translation_enabled = Column(Boolean)
|
|
is_translator = Column(Boolean)
|
|
lang = Column(Text)
|
|
listed_count = Column(Integer)
|
|
location = Column(Text)
|
|
name = Column(Text)
|
|
notifications = Column(Boolean)
|
|
profile_background_color = Column(Text)
|
|
profile_background_image_url = Column(Text)
|
|
profile_background_image_url_https = Column(Text)
|
|
profile_background_tile = Column(Boolean)
|
|
profile_banner_url = Column(Text)
|
|
profile_image_url = Column(Text)
|
|
profile_image_url_https = Column(Text)
|
|
profile_link_color = Column(Text)
|
|
profile_sidebar_border_color = Column(Text)
|
|
profile_sidebar_fill_color = Column(Text)
|
|
profile_text_color = Column(Text)
|
|
profile_use_background_image = Column(Boolean)
|
|
protected = Column(Boolean)
|
|
screen_name = Column(Text)
|
|
statuses_count = Column(Integer)
|
|
time_zone = Column(Text)
|
|
url = Column(Text)
|
|
utc_offset = Column(Integer)
|
|
verified = Column(Boolean)
|
|
|
|
|
|
def as_dict(self):
|
|
dcopy = self.__dict__.copy()
|
|
for k,v in self.__dict__.items():
|
|
if k[0] == '_':
|
|
del dcopy[k]
|
|
try:
|
|
dcopy['entities'] = json.loads(dcopy['entities'])
|
|
except Exception:
|
|
print('Could not convert to dict')
|
|
pass
|
|
return dcopy
|
|
|
|
class Following(Base):
|
|
__tablename__ = 'followers'
|
|
|
|
id = Column(Integer, primary_key=True, autoincrement=True)
|
|
isfollowed = Column(Integer)
|
|
follower = Column(Integer)
|
|
created_at_stamp = Column(Text)
|
|
|
|
follower_index = Index('isfollowed', 'follower')
|
|
|
|
class ExtractorEntry(Base):
|
|
__tablename__ = 'extractor-cursor'
|
|
|
|
id = Column(Integer, primary_key=True, default=lambda x: int(time.time()*1000))
|
|
user = Column(BigInteger, index=True)
|
|
cursor = Column(BigInteger, default=-1)
|
|
pending = Column(Boolean, default=False)
|
|
errors = Column(Text, default="")
|
|
busy = Column(Boolean, default=False)
|
|
|
|
|
|
class Search(Base):
|
|
__tablename__ = 'search_queries'
|
|
|
|
id = Column(Integer, primary_key=True, index=True, unique=True)
|
|
endpoint = Column(Text, comment="Endpoint URL")
|
|
attrs = Column(Text, comment="Text version of the dictionary of parameters")
|
|
count = Column(Integer)
|
|
current_count = Column(Integer)
|
|
current_id = Column(BigInteger, comment='Oldest ID retrieved (should match max_id when done)')
|
|
since_id = Column(BigInteger)
|
|
|
|
class SearchResults(Base):
|
|
__tablename__ = 'search_results'
|
|
id = Column(Integer, primary_key=True, index=True, unique=True)
|
|
search_id = Column(ForeignKey('search_queries.id'))
|
|
resource_id = Column(Text)
|
|
|
|
def memoize(f):
|
|
memo = {}
|
|
@wraps(f)
|
|
def helper(self, **kwargs):
|
|
st = dict_to_str(kwargs)
|
|
key = (self.__uriparts, st)
|
|
if key not in memo:
|
|
memo[key] = f(self, **kwargs)
|
|
return memo[key]
|
|
return helper
|
|
|
|
def make_session(url):
|
|
if not isinstance(url, str):
|
|
print(url)
|
|
raise Exception("FUCK")
|
|
engine = create_engine(url, poolclass=SingletonThreadPool)#, echo=True)
|
|
Base.metadata.create_all(engine)
|
|
Session = sessionmaker(bind=engine)
|
|
session = Session()
|
|
return session
|
|
|
|
|
|
def dict_to_str(args):
|
|
return json.dumps(args, sort_keys=True)
|