From 75edbf94b9e948cdb2c83a4ca1aa2e31a4ae5624 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Wed, 23 Nov 2022 18:57:53 +0100 Subject: [PATCH] Mitigated performance issues related to SQLite3 randomly hanging for a second on Android --- sbapp/main.py | 92 +++++++--- sbapp/sideband/core.py | 363 ++++++++++++++++++++++---------------- sbapp/ui/conversations.py | 4 - sbapp/ui/messages.py | 60 ++++--- 4 files changed, 322 insertions(+), 197 deletions(-) diff --git a/sbapp/main.py b/sbapp/main.py index b784964..ae4210f 100644 --- a/sbapp/main.py +++ b/sbapp/main.py @@ -143,7 +143,7 @@ class SidebandApp(MDApp): self.check_bluetooth_permissions() self.start_service() - Clock.schedule_interval(self.jobs, 1) + Clock.schedule_interval(self.jobs, 1.5) def dismiss_splash(dt): from android import loadingscreen @@ -521,18 +521,40 @@ class SidebandApp(MDApp): return screen def jobs(self, delta_time): + # TODO: Remove + js = time.time() + actions = [] + if self.root.ids.screen_manager.current == "messages_screen": + # TODO: Remove + actions.append("messages_screen") + self.messages_view.update() if not self.root.ids.messages_scrollview.dest_known: + # TODO: Remove + actions.append("messages_area_detect") + self.message_area_detect() elif self.root.ids.screen_manager.current == "conversations_screen": - if self.sideband.getstate("app.flags.unread_conversations"): + # TODO: Remove + actions.append("conversations_screen") + + if self.sideband.getstate("app.flags.unread_conversations", allow_cache=True): + # TODO: Remove + actions.append("unread_conversations") + if self.conversations_view != None: + # TODO: Remove + actions.append("conversations_view.update") + self.conversations_view.update() - if self.sideband.getstate("app.flags.lxmf_sync_dialog_open") and self.sync_dialog != None: + if self.sideband.getstate("app.flags.lxmf_sync_dialog_open", allow_cache=True) and self.sync_dialog != None: + # TODO: Remove + actions.append("lxmf_sync_dialog_open") + self.sync_dialog.ids.sync_progress.value = self.sideband.get_sync_progress()*100 self.sync_dialog.ids.sync_status.text = self.sideband.get_sync_status() @@ -543,20 +565,39 @@ class SidebandApp(MDApp): self.widget_hide(self.sync_dialog.stop_button, True) elif self.root.ids.screen_manager.current == "announces_screen": - if self.sideband.getstate("app.flags.new_announces"): + # TODO: Remove + actions.append("announces_screen") + + if self.sideband.getstate("app.flags.new_announces", allow_cache=True): + # TODO: Remove + actions.append("new_announces") + if self.announces_view != None: + actions.append("announces_view.update") self.announces_view.update() - if self.sideband.getstate("app.flags.new_conversations"): + if self.sideband.getstate("app.flags.new_conversations", allow_cache=True): + # TODO: Remove + actions.append("new_conversations") + + if self.conversations_view != None: + # TODO: Remove + actions.append("conversations_view.update") + + self.conversations_view.update() + + if self.sideband.getstate("wants.viewupdate.conversations", allow_cache=True): + # TODO: Remove + actions.append("wants.viewupdate.conversations") + if self.conversations_view != None: self.conversations_view.update() - if self.sideband.getstate("wants.viewupdate.conversations"): - if self.conversations_view != None: - self.conversations_view.update() - - if self.sideband.getstate("lxm_uri_ingest.result"): - info_text = self.sideband.getstate("lxm_uri_ingest.result") + if self.sideband.getstate("lxm_uri_ingest.result", allow_cache=True): + # TODO: Remove + actions.append("lxm_uri_ingest.result") + + info_text = self.sideband.getstate("lxm_uri_ingest.result", allow_cache=True) self.sideband.setstate("lxm_uri_ingest.result", False) ok_button = MDRectangleFlatButton(text="OK",font_size=dp(18)) dialog = MDDialog( @@ -571,6 +612,12 @@ class SidebandApp(MDApp): ok_button.bind(on_release=dl_ok) dialog.open() + # TODO: Remove Timing and Profiling + jd = time.time()-js + if jd > 0.25: + RNS.log("Jobs completed in "+RNS.prettytime(jd), RNS.LOG_DEBUG) + RNS.log(str(actions)) + def on_start(self): self.last_exit_event = time.time() self.root.ids.screen_manager.transition.duration = 0.25 @@ -691,7 +738,7 @@ class SidebandApp(MDApp): MDApp.get_running_app().stop() Window.close() - Clock.schedule_once(final_exit, 0.65) + Clock.schedule_once(final_exit, 0.85) def announce_now_action(self, sender=None): self.sideband.lxmf_announce() @@ -927,7 +974,6 @@ class SidebandApp(MDApp): def connectivity_status(self, sender): hs = dp(22) - yes_button = MDRectangleFlatButton(text="OK",font_size=dp(18)) dialog = MDDialog( title="Connectivity Status", @@ -940,12 +986,16 @@ class SidebandApp(MDApp): def dl_yes(s): self.connectivity_updater.cancel() dialog.dismiss() + if self.connectivity_updater != None: + self.connectivity_updater.cancel() yes_button.bind(on_release=dl_yes) dialog.open() + if self.connectivity_updater != None: self.connectivity_updater.cancel() - self.connectivity_updater = Clock.schedule_interval(cs_updater, 1.0) + + self.connectivity_updater = Clock.schedule_interval(cs_updater, 2.0) def ingest_lxm_action(self, sender): def cb(dt): @@ -1136,13 +1186,11 @@ class SidebandApp(MDApp): ### Settings screen ###################################### def settings_action(self, sender=None): + self.settings_init() + self.root.ids.screen_manager.transition.direction = "left" self.root.ids.nav_drawer.set_state("closed") - def cb(dt): - self.root.ids.screen_manager.transition.direction = "left" - self.settings_init() - self.root.ids.screen_manager.current = "settings_screen" - self.sideband.setstate("app.displaying", self.root.ids.screen_manager.current) - Clock.schedule_once(cb, 0.2) + self.root.ids.screen_manager.current = "settings_screen" + self.sideband.setstate("app.displaying", self.root.ids.screen_manager.current) def settings_init(self, sender=None): if not self.settings_ready: @@ -1301,10 +1349,10 @@ class SidebandApp(MDApp): ### Connectivity screen ###################################### def connectivity_action(self, sender=None): - self.connectivity_init() self.root.ids.screen_manager.transition.direction = "left" - self.root.ids.screen_manager.current = "connectivity_screen" self.root.ids.nav_drawer.set_state("closed") + self.connectivity_init() + self.root.ids.screen_manager.current = "connectivity_screen" self.sideband.setstate("app.displaying", self.root.ids.screen_manager.current) def connectivity_init(self, sender=None): diff --git a/sbapp/sideband/core.py b/sbapp/sideband/core.py index 88dba1a..060adac 100644 --- a/sbapp/sideband/core.py +++ b/sbapp/sideband/core.py @@ -22,35 +22,36 @@ class PropagationNodeDetector(): def received_announce(self, destination_hash, announced_identity, app_data): try: - unpacked = msgpack.unpackb(app_data) - node_active = unpacked[0] - emitted = unpacked[1] - hops = RNS.Transport.hops_to(destination_hash) + if len(app_data) > 0: + unpacked = msgpack.unpackb(app_data) + node_active = unpacked[0] + emitted = unpacked[1] + hops = RNS.Transport.hops_to(destination_hash) - age = time.time() - emitted - if age < 0: - RNS.log("Warning, propagation node announce emitted in the future, possible timing inconsistency or tampering attempt.") - if age < -1*PropagationNodeDetector.EMITTED_DELTA_GRACE: - raise ValueError("Announce timestamp too far in the future, discarding it") + age = time.time() - emitted + if age < 0: + RNS.log("Warning, propagation node announce emitted in the future, possible timing inconsistency or tampering attempt.") + if age < -1*PropagationNodeDetector.EMITTED_DELTA_GRACE: + raise ValueError("Announce timestamp too far in the future, discarding it") - if age > -1*PropagationNodeDetector.EMITTED_DELTA_IGNORE: - # age = 0 - pass + if age > -1*PropagationNodeDetector.EMITTED_DELTA_IGNORE: + # age = 0 + pass - RNS.log("Detected active propagation node "+RNS.prettyhexrep(destination_hash)+" emission "+str(age)+" seconds ago, "+str(hops)+" hops away") - self.owner.log_announce(destination_hash, RNS.prettyhexrep(destination_hash).encode("utf-8"), dest_type=PropagationNodeDetector.aspect_filter) + RNS.log("Detected active propagation node "+RNS.prettyhexrep(destination_hash)+" emission "+str(age)+" seconds ago, "+str(hops)+" hops away") + self.owner.log_announce(destination_hash, RNS.prettyhexrep(destination_hash).encode("utf-8"), dest_type=PropagationNodeDetector.aspect_filter) - if self.owner.config["lxmf_propagation_node"] == None: - if self.owner.active_propagation_node == None: - self.owner.set_active_propagation_node(destination_hash) - else: - prev_hops = RNS.Transport.hops_to(self.owner.active_propagation_node) - if hops <= prev_hops: + if self.owner.config["lxmf_propagation_node"] == None: + if self.owner.active_propagation_node == None: self.owner.set_active_propagation_node(destination_hash) else: - pass - else: - pass + prev_hops = RNS.Transport.hops_to(self.owner.active_propagation_node) + if hops <= prev_hops: + self.owner.set_active_propagation_node(destination_hash) + else: + pass + else: + pass except Exception as e: RNS.log("Error while processing received propagation node announce: "+str(e)) @@ -79,6 +80,7 @@ class SidebandCore(): def __init__(self, owner_app, is_service=False, is_client=False, android_app_dir=None, verbose=False): self.is_service = is_service self.is_client = is_client + self.db = None if not self.is_service and not self.is_client: self.is_standalone = True @@ -126,6 +128,8 @@ class SidebandCore(): self.first_run = True self.saving_configuration = False + self.getstate_cache = {} + try: if not os.path.isfile(self.config_path): self.__init_config() @@ -399,6 +403,7 @@ class SidebandCore(): else: self._db_initstate() self._db_initpersistent() + self.__db_indices() def __reload_config(self): RNS.log("Reloading Sideband configuration... "+str(self.config_path), RNS.LOG_DEBUG) @@ -626,13 +631,54 @@ class SidebandCore(): return self._db_getstate("app.active_conversation") def setstate(self, prop, val): + self.getstate_cache[prop] = val self._db_setstate(prop, val) + # def cb(): + # self._db_setstate(prop, val) + # threading.Thread(target=cb, daemon=True).start() + + def getstate(self, prop, allow_cache=False): + if not RNS.vendor.platformutils.is_android(): + return self._db_getstate(prop) + + else: + db_timeout = 0.060 + cached_value = None + has_cached_value = False + if prop in self.getstate_cache: + cached_value = self.getstate_cache[prop] + has_cached_value = True + + if not allow_cache or not has_cached_value: + self.getstate_cache[prop] = self._db_getstate(prop) + return self.getstate_cache[prop] + + else: + get_thread_running = True + def get_job(): + self.getstate_cache[prop] = self._db_getstate(prop) + get_thread_running = False + + get_thread = threading.Thread(target=get_job, daemon=True) + get_thread.timeout = time.time()+db_timeout + get_thread.start() + + while get_thread.is_alive() and time.time() < get_thread.timeout: + time.sleep(0.01) + + if get_thread.is_alive(): + return self.getstate_cache[prop] + else: + return self.getstate_cache[prop] + + - def getstate(self, prop): - return self._db_getstate(prop) def setpersistent(self, prop, val): self._db_setpersistent(prop, val) + # def cb(): + # self._db_setpersistent(prop, val) + # threading.Thread(target=cb, daemon=True).start() def getpersistent(self, prop): return self._db_getpersistent(prop) @@ -643,8 +689,14 @@ class SidebandCore(): def __event_conversation_changed(self, context_dest): pass + def __db_connect(self): + if self.db == None: + self.db = sqlite3.connect(self.db_path, check_same_thread=False) + + return self.db + def __db_init(self): - db = sqlite3.connect(self.db_path) + db = self.__db_connect() dbc = db.cursor() dbc.execute("DROP TABLE IF EXISTS lxm") @@ -663,105 +715,153 @@ class SidebandCore(): dbc.execute("CREATE TABLE persistent (property BLOB PRIMARY KEY, value BLOB)") db.commit() - db.close() + + def __db_indices(self): + db = self.__db_connect() + dbc = db.cursor() + dbc.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_persistent_property ON persistent(property)") + dbc.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_state_property ON state(property)") + dbc.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_conv_dest_context ON conv(dest_context)") + db.commit() def _db_initstate(self): - db = sqlite3.connect(self.db_path) + db = self.__db_connect() dbc = db.cursor() dbc.execute("DROP TABLE IF EXISTS state") dbc.execute("CREATE TABLE state (property BLOB PRIMARY KEY, value BLOB)") + db.commit() self._db_setstate("database_ready", True) def _db_getstate(self, prop): - db = sqlite3.connect(self.db_path) - dbc = db.cursor() - - query = "select * from state where property=:uprop" - dbc.execute(query, {"uprop": prop.encode("utf-8")}) - result = dbc.fetchall() - db.close() + try: + db = self.__db_connect() + dbc = db.cursor() - if len(result) < 1: - return None - else: - try: - entry = result[0] - val = msgpack.unpackb(entry[1]) - return val - except Exception as e: - RNS.log("Could not unpack state value from database for property \""+str(prop)+"\". The contained exception was: "+str(e), RNS.LOG_ERROR) + query = "select * from state where property=:uprop" + dbc.execute(query, {"uprop": prop.encode("utf-8")}) + + result = dbc.fetchall() + + if len(result) < 1: return None + else: + try: + entry = result[0] + val = msgpack.unpackb(entry[1]) + + return val + except Exception as e: + RNS.log("Could not unpack state value from database for property \""+str(prop)+"\". The contained exception was: "+str(e), RNS.LOG_ERROR) + return None + + except Exception as e: + RNS.log("An error occurred during getstate database operation: "+str(e), RNS.LOG_ERROR) + self.db = None def _db_setstate(self, prop, val): - db = sqlite3.connect(self.db_path) - dbc = db.cursor() - uprop = prop.encode("utf-8") - bval = msgpack.packb(val) + try: + uprop = prop.encode("utf-8") + bval = msgpack.packb(val) - if self._db_getstate(prop) == None: - query = "INSERT INTO state (property, value) values (?, ?)" - data = (uprop, bval) - dbc.execute(query, data) - else: - query = "UPDATE state set value=:bval where property=:uprop;" - dbc.execute(query, {"bval": bval, "uprop": uprop}) + if self._db_getstate(prop) == None: + try: + db = self.__db_connect() + dbc = db.cursor() + query = "INSERT INTO state (property, value) values (?, ?)" + data = (uprop, bval) + dbc.execute(query, data) + db.commit() - db.commit() - db.close() + except Exception as e: + RNS.log("Error while setting state property "+str(prop)+" in DB: "+str(e), RNS.LOG_ERROR) + RNS.log("Retrying as update query...", RNS.LOG_ERROR) + db = self.__db_connect() + dbc = db.cursor() + query = "UPDATE state set value=:bval where property=:uprop;" + dbc.execute(query, {"bval": bval, "uprop": uprop}) + db.commit() + + else: + db = self.__db_connect() + dbc = db.cursor() + query = "UPDATE state set value=:bval where property=:uprop;" + dbc.execute(query, {"bval": bval, "uprop": uprop}) + db.commit() + + + except Exception as e: + RNS.log("An error occurred during setstate database operation: "+str(e), RNS.LOG_ERROR) + self.db = None def _db_initpersistent(self): - db = sqlite3.connect(self.db_path) + db = self.__db_connect() dbc = db.cursor() dbc.execute("CREATE TABLE IF NOT EXISTS persistent (property BLOB PRIMARY KEY, value BLOB)") + db.commit() def _db_getpersistent(self, prop): - db = sqlite3.connect(self.db_path) - dbc = db.cursor() - - query = "select * from persistent where property=:uprop" - dbc.execute(query, {"uprop": prop.encode("utf-8")}) - result = dbc.fetchall() - db.close() + try: + db = self.__db_connect() + dbc = db.cursor() + + query = "select * from persistent where property=:uprop" + dbc.execute(query, {"uprop": prop.encode("utf-8")}) + result = dbc.fetchall() - if len(result) < 1: - return None - else: - try: - entry = result[0] - val = msgpack.unpackb(entry[1]) - if val == None: - db = sqlite3.connect(self.db_path) - dbc = db.cursor() - query = "delete from persistent where (property=:uprop);" - dbc.execute(query, {"uprop": prop.encode("utf-8")}) - db.commit() - - return val - except Exception as e: - RNS.log("Could not unpack persistent value from database for property \""+str(prop)+"\". The contained exception was: "+str(e), RNS.LOG_ERROR) + if len(result) < 1: return None + else: + try: + entry = result[0] + val = msgpack.unpackb(entry[1]) + if val == None: + query = "delete from persistent where (property=:uprop);" + dbc.execute(query, {"uprop": prop.encode("utf-8")}) + db.commit() + + return val + except Exception as e: + RNS.log("Could not unpack persistent value from database for property \""+str(prop)+"\". The contained exception was: "+str(e), RNS.LOG_ERROR) + return None + + except Exception as e: + RNS.log("An error occurred during persistent getstate database operation: "+str(e), RNS.LOG_ERROR) + self.db = None def _db_setpersistent(self, prop, val): - db = sqlite3.connect(self.db_path) - dbc = db.cursor() - uprop = prop.encode("utf-8") - bval = msgpack.packb(val) + try: + db = self.__db_connect() + dbc = db.cursor() + uprop = prop.encode("utf-8") + bval = msgpack.packb(val) - if self._db_getpersistent(prop) == None: - query = "INSERT INTO persistent (property, value) values (?, ?)" - data = (uprop, bval) - dbc.execute(query, data) - else: - query = "UPDATE persistent set value=:bval where property=:uprop;" - dbc.execute(query, {"bval": bval, "uprop": uprop}) + if self._db_getpersistent(prop) == None: + try: + query = "INSERT INTO persistent (property, value) values (?, ?)" + data = (uprop, bval) + dbc.execute(query, data) + db.commit() + + except Exception as e: + RNS.log("Error while setting persistent state property "+str(prop)+" in DB: "+str(e), RNS.LOG_ERROR) + RNS.log("Retrying as update query...") + query = "UPDATE state set value=:bval where property=:uprop;" + dbc.execute(query, {"bval": bval, "uprop": uprop}) + db.commit() - db.commit() - db.close() + else: + query = "UPDATE persistent set value=:bval where property=:uprop;" + dbc.execute(query, {"bval": bval, "uprop": uprop}) + db.commit() + + except Exception as e: + RNS.log("An error occurred during persistent setstate database operation: "+str(e), RNS.LOG_ERROR) + self.db = None def _db_conversation_set_unread(self, context_dest, unread): - db = sqlite3.connect(self.db_path) + db = self.__db_connect() dbc = db.cursor() query = "UPDATE conv set unread = ? where dest_context = ?" @@ -770,10 +870,8 @@ class SidebandCore(): result = dbc.fetchall() db.commit() - db.close() - def _db_conversation_set_trusted(self, context_dest, trusted): - db = sqlite3.connect(self.db_path) + db = self.__db_connect() dbc = db.cursor() query = "UPDATE conv set trust = ? where dest_context = ?" @@ -782,10 +880,8 @@ class SidebandCore(): result = dbc.fetchall() db.commit() - db.close() - def _db_conversation_set_name(self, context_dest, name): - db = sqlite3.connect(self.db_path) + db = self.__db_connect() dbc = db.cursor() query = "UPDATE conv set name=:name_data where dest_context=:ctx;" @@ -793,17 +889,13 @@ class SidebandCore(): result = dbc.fetchall() db.commit() - db.close() - def _db_conversations(self): - db = sqlite3.connect(self.db_path) + db = self.__db_connect() dbc = db.cursor() dbc.execute("select * from conv") result = dbc.fetchall() - db.close() - if len(result) < 1: return None else: @@ -818,14 +910,12 @@ class SidebandCore(): return convs def _db_announces(self): - db = sqlite3.connect(self.db_path) + db = self.__db_connect() dbc = db.cursor() dbc.execute("select * from announce order by received desc") result = dbc.fetchall() - db.close() - if len(result) < 1: return None else: @@ -849,15 +939,13 @@ class SidebandCore(): return announces def _db_conversation(self, context_dest): - db = sqlite3.connect(self.db_path) + db = self.__db_connect() dbc = db.cursor() query = "select * from conv where dest_context=:ctx" dbc.execute(query, {"ctx": context_dest}) result = dbc.fetchall() - db.close() - if len(result) < 1: return None else: @@ -875,40 +963,35 @@ class SidebandCore(): def _db_clear_conversation(self, context_dest): RNS.log("Clearing conversation with "+RNS.prettyhexrep(context_dest), RNS.LOG_DEBUG) - db = sqlite3.connect(self.db_path) + db = self.__db_connect() dbc = db.cursor() query = "delete from lxm where (dest=:ctx_dst or source=:ctx_dst);" dbc.execute(query, {"ctx_dst": context_dest}) db.commit() - db.close() - def _db_delete_conversation(self, context_dest): RNS.log("Deleting conversation with "+RNS.prettyhexrep(context_dest), RNS.LOG_DEBUG) - db = sqlite3.connect(self.db_path) + db = self.__db_connect() dbc = db.cursor() query = "delete from conv where (dest_context=:ctx_dst);" dbc.execute(query, {"ctx_dst": context_dest}) db.commit() - db.close() def _db_delete_announce(self, context_dest): RNS.log("Deleting announce with "+RNS.prettyhexrep(context_dest), RNS.LOG_DEBUG) - db = sqlite3.connect(self.db_path) + db = self.__db_connect() dbc = db.cursor() query = "delete from announce where (source=:ctx_dst);" dbc.execute(query, {"ctx_dst": context_dest}) db.commit() - db.close() - def _db_create_conversation(self, context_dest, name = None, trust = False): RNS.log("Creating conversation for "+RNS.prettyhexrep(context_dest), RNS.LOG_DEBUG) - db = sqlite3.connect(self.db_path) + db = self.__db_connect() dbc = db.cursor() def_name = "".encode("utf-8") @@ -916,9 +999,7 @@ class SidebandCore(): data = (context_dest, 0, 0, 0, SidebandCore.CONV_P2P, 0, def_name, msgpack.packb(None)) dbc.execute(query, data) - db.commit() - db.close() if trust: self._db_conversation_set_trusted(context_dest, True) @@ -930,28 +1011,24 @@ class SidebandCore(): def _db_delete_message(self, msg_hash): RNS.log("Deleting message "+RNS.prettyhexrep(msg_hash)) - db = sqlite3.connect(self.db_path) + db = self.__db_connect() dbc = db.cursor() query = "delete from lxm where (lxm_hash=:mhash);" dbc.execute(query, {"mhash": msg_hash}) db.commit() - db.close() - def _db_clean_messages(self): RNS.log("Purging stale messages... "+str(self.db_path)) - db = sqlite3.connect(self.db_path) + db = self.__db_connect() dbc = db.cursor() query = "delete from lxm where (state=:outbound_state or state=:sending_state);" dbc.execute(query, {"outbound_state": LXMF.LXMessage.OUTBOUND, "sending_state": LXMF.LXMessage.SENDING}) db.commit() - db.close() - def _db_message_set_state(self, lxm_hash, state): - db = sqlite3.connect(self.db_path) + db = self.__db_connect() dbc = db.cursor() query = "UPDATE lxm set state = ? where lxm_hash = ?" @@ -960,10 +1037,8 @@ class SidebandCore(): db.commit() result = dbc.fetchall() - db.close() - def _db_message_set_method(self, lxm_hash, method): - db = sqlite3.connect(self.db_path) + db = self.__db_connect() dbc = db.cursor() query = "UPDATE lxm set method = ? where lxm_hash = ?" @@ -972,21 +1047,17 @@ class SidebandCore(): db.commit() result = dbc.fetchall() - db.close() - def message(self, msg_hash): return self._db_message(msg_hash) def _db_message(self, msg_hash): - db = sqlite3.connect(self.db_path) + db = self.__db_connect() dbc = db.cursor() query = "select * from lxm where lxm_hash=:mhash" dbc.execute(query, {"mhash": msg_hash}) result = dbc.fetchall() - db.close() - if len(result) < 1: return None else: @@ -1020,7 +1091,7 @@ class SidebandCore(): return message def _db_message_count(self, context_dest): - db = sqlite3.connect(self.db_path) + db = self.__db_connect() dbc = db.cursor() query = "select count(*) from lxm where dest=:context_dest or source=:context_dest" @@ -1028,15 +1099,13 @@ class SidebandCore(): result = dbc.fetchall() - db.close() - if len(result) < 1: return None else: return result[0][0] def _db_messages(self, context_dest, after = None, before = None, limit = None): - db = sqlite3.connect(self.db_path) + db = self.__db_connect() dbc = db.cursor() if after != None and before == None: @@ -1054,8 +1123,6 @@ class SidebandCore(): result = dbc.fetchall() - db.close() - if len(result) < 1: return None else: @@ -1095,7 +1162,7 @@ class SidebandCore(): def _db_save_lxm(self, lxm, context_dest): state = lxm.state - db = sqlite3.connect(self.db_path) + db = self.__db_connect() dbc = db.cursor() if not lxm.packed: @@ -1124,12 +1191,11 @@ class SidebandCore(): dbc.execute(query, data) db.commit() - db.close() self.__event_conversation_changed(context_dest) def _db_save_announce(self, destination_hash, app_data, dest_type="lxmf.delivery"): - db = sqlite3.connect(self.db_path) + db = self.__db_connect() dbc = db.cursor() query = "delete from announce where id is NULL or id not in (select id from announce order by received desc limit "+str(self.MAX_ANNOUNCES)+")" @@ -1154,9 +1220,6 @@ class SidebandCore(): dbc.execute(query, data) db.commit() - db.commit() - db.close() - def lxmf_announce(self): if self.is_standalone or self.is_service: self.lxmf_destination.announce() diff --git a/sbapp/ui/conversations.py b/sbapp/ui/conversations.py index ed0410d..6d158b2 100644 --- a/sbapp/ui/conversations.py +++ b/sbapp/ui/conversations.py @@ -99,8 +99,6 @@ class Conversations(): unread = conv["unread"] if not context_dest in self.added_item_dests: - i_s = time.time() - iconl = IconLeftWidget(icon=self.trust_icon(context_dest, unread), on_release=self.app.conversation_action) item = OneLineAvatarIconListItem(text=self.app.sideband.peer_display_name(context_dest), on_release=self.app.conversation_action) item.add_widget(iconl) @@ -280,8 +278,6 @@ class Conversations(): self.added_item_dests.append(context_dest) self.list.add_widget(item) - RNS.log("Created item in "+RNS.prettytime(time.time()-i_s), RNS.LOG_DEBUG) - else: for w in self.list.children: if w.sb_uid == context_dest: diff --git a/sbapp/ui/messages.py b/sbapp/ui/messages.py index f8fc3f2..d1c7b46 100644 --- a/sbapp/ui/messages.py +++ b/sbapp/ui/messages.py @@ -37,9 +37,12 @@ class Messages(): def __init__(self, app, context_dest): self.app = app self.context_dest = context_dest - self.messages = [] + self.new_messages = [] self.added_item_hashes = [] + self.added_messages = 0 self.latest_message_timestamp = None + self.earliest_message_timestamp = time.time() + self.loading_earlier_messages = False self.list = None self.widgets = [] self.send_error_dialog = None @@ -50,20 +53,27 @@ class Messages(): if self.list != None: self.list.clear_widgets() - self.messages = [] + self.new_messages = [] self.added_item_hashes = [] + self.added_messages = 0 self.latest_message_timestamp = None self.widgets = [] self.update() + def load_more(self, dt): + for new_message in self.app.sideband.list_messages(self.context_dest, before=self.earliest_message_timestamp,limit=2): + self.new_messages.append(new_message) + + if len(self.new_messages) > 0: + self.loading_earlier_messages = True + self.list.remove_widget(self.load_more_button) + def update(self, limit=8): - s_ts = time.time() - self.messages = self.app.sideband.list_messages(self.context_dest, after=self.latest_message_timestamp,limit=limit) + for new_message in self.app.sideband.list_messages(self.context_dest, after=self.latest_message_timestamp,limit=limit): + self.new_messages.append(new_message) self.db_message_count = self.app.sideband.count_messages(self.context_dest) - RNS.log("Total messages in db: "+str(self.db_message_count)) - RNS.log("Added items: "+str(len(self.added_item_hashes))) if self.load_more_button == None: self.load_more_button = MDRectangleFlatIconButton( @@ -73,29 +83,28 @@ class Messages(): theme_text_color="Custom", size_hint=[1.0, None], ) + def lmcb(sender): + Clock.schedule_once(self.load_more, 0.15) + + self.load_more_button.bind(on_release=lmcb) if self.list == None: layout = GridLayout(cols=1, spacing=dp(16), padding=dp(16), size_hint_y=None) layout.bind(minimum_height=layout.setter('height')) self.list = layout - - if (len(self.added_item_hashes) < self.db_message_count) and not self.load_more_button in self.list.children: - # if self.load_more_button in self.list.children: - # RNS.log("Removing for reinsertion") - # self.list.remove_widget(self.load_more_button) - self.list.add_widget(self.load_more_button, len(self.list.children)) c_ts = time.time() - if len(self.messages) > 0: + if len(self.new_messages) > 0: self.update_widget() - RNS.log("Cards created in "+RNS.prettytime(time.time()-c_ts), RNS.LOG_DEBUG) + + if (len(self.added_item_hashes) < self.db_message_count) and not self.load_more_button in self.list.children: + self.list.add_widget(self.load_more_button, len(self.list.children)) if self.app.sideband.config["dark_ui"]: intensity_msgs = intensity_msgs_dark else: intensity_msgs = intensity_msgs_light - upd_ts = time.time() for w in self.widgets: m = w.m if self.app.sideband.config["dark_ui"]: @@ -141,8 +150,6 @@ class Messages(): w.heading = titlestr+"[b]Sent[/b] "+txstr+"\n[b]State[/b] Failed" m["state"] = msg["state"] - RNS.log("Updated message widgets in "+RNS.prettytime(time.time()-upd_ts), RNS.LOG_DEBUG) - RNS.log("Creating messages view took "+RNS.prettytime(time.time()-s_ts), RNS.LOG_DEBUG) def update_widget(self): if self.app.sideband.config["dark_ui"]: @@ -152,8 +159,10 @@ class Messages(): intensity_msgs = intensity_msgs_light mt_color = [1.0, 1.0, 1.0, 0.95] - for m in self.messages: - s_ts = time.time() + if self.loading_earlier_messages: + self.new_messages.reverse() + + for m in self.new_messages: if not m["hash"] in self.added_item_hashes: txstr = time.strftime(ts_format, time.localtime(m["sent"])) rxstr = time.strftime(ts_format, time.localtime(m["received"])) @@ -432,14 +441,23 @@ class Messages(): # Bind menu open item.ids.msg_submenu.bind(on_release=callback_factory(item)) + if self.loading_earlier_messages: + insert_pos = len(self.list.children) + else: + insert_pos = 0 + self.added_item_hashes.append(m["hash"]) self.widgets.append(item) - self.list.add_widget(item) + self.list.add_widget(item, insert_pos) if self.latest_message_timestamp == None or m["received"] > self.latest_message_timestamp: self.latest_message_timestamp = m["received"] - RNS.log("Created message card in "+RNS.prettytime(time.time()-s_ts), RNS.LOG_DEBUG) + if self.earliest_message_timestamp == None or m["received"] < self.earliest_message_timestamp: + self.earliest_message_timestamp = m["received"] + + self.added_messages += len(self.new_messages) + self.new_messages = [] def get_widget(self): return self.list