jacob.eva/Sideband_CE
1
0
Fork 0
mirror of https://github.com/liberatedsystems/Sideband_CE.git synced 2024-09-03 04:13:27 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

Implemented bulk telemetry transfers

Browse Source
This commit is contained in:
Mark Qvist 2023-10-30 22:26:20 +01:00
parent 08e112f538
commit 26ebc80a22
4 changed files with 165 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
sbapp/main.py
View File

@ -3207,7 +3207,14 @@ class SidebandApp(MDApp):
self.telemetry_info_dialog.dismiss() self.telemetry_info_dialog.dismiss()
ok_button.bind(on_release=dl_ok) ok_button.bind(on_release=dl_ok)
result = self.sideband.send_latest_telemetry(to_addr=self.sideband.config["telemetry_collector"]) collector_address = self.sideband.config["telemetry_collector"]
if self.sideband.config["telemetry_send_all_to_collector"]:
last_timebase = (self.sideband.getpersistent(f"telemetry.{RNS.hexrep(collector_address, delimit=False)}.last_send_success_timebase") or 0)
result = self.sideband.create_telemetry_collector_response(to_addr=collector_address, timebase=last_timebase, is_authorized_telemetry_request=True)
else:
result = self.sideband.send_latest_telemetry(to_addr=collector_address)
if result == "no_address": if result == "no_address":
title_str = "Invalid Address" title_str = "Invalid Address"
info_str = "You must specify a valid LXMF address for the collector you want to sent data to." info_str = "You must specify a valid LXMF address for the collector you want to sent data to."
@ -3223,9 +3230,15 @@ class SidebandApp(MDApp):
elif result == "sent": elif result == "sent":
title_str = "Update Sent" title_str = "Update Sent"
info_str = "A telemetry update was sent to the collector." info_str = "A telemetry update was sent to the collector."
elif result == "not_sent":
title_str = "Not Sent"
info_str = "The telemetry update could not be sent."
elif result == "nothing_to_send":
title_str = "Nothing to Send"
info_str = "There was no new data to send."
else: else:
title_str = "Unknown Status" title_str = "Unknown Status"
info_str = "The status of the telemetry update is unknown." info_str = "The status of the telemetry update is unknown: "+str(result)
self.telemetry_info_dialog.title = title_str self.telemetry_info_dialog.title = title_str
self.telemetry_info_dialog.text = info_str self.telemetry_info_dialog.text = info_str
@ -3263,7 +3276,7 @@ class SidebandApp(MDApp):
info_str = "A telemetry request could not be sent." info_str = "A telemetry request could not be sent."
else: else:
title_str = "Unknown Status" title_str = "Unknown Status"
info_str = "The status of the telemetry request is unknown." info_str = "The status of the telemetry request is unknown: "+str(result)
self.telemetry_info_dialog.title = title_str self.telemetry_info_dialog.title = title_str
self.telemetry_info_dialog.text = info_str self.telemetry_info_dialog.text = info_str

126
sbapp/sideband/core.py
View File

@ -747,21 +747,25 @@ class SidebandCore():
RNS.log("Error while checking telemetry sending for "+RNS.prettyhexrep(context_dest)+": "+str(e), RNS.LOG_ERROR) RNS.log("Error while checking telemetry sending for "+RNS.prettyhexrep(context_dest)+": "+str(e), RNS.LOG_ERROR)
return False return False
def requests_allowed_from(self, context_dest): def allow_request_from(self, context_dest):
try: try:
if self.config["telemetry_allow_requests_from_anyone"] == True: if self.config["telemetry_allow_requests_from_anyone"] == True:
return True return True
if self.config["telemetry_allow_requests_from_trusted"] == True:
existing_conv = self._db_conversation(context_dest)
return existing_conv["trust"] == 1
return self.requests_allowed_from(context_dest)
except Exception as e:
RNS.log("Error while checking request permissions for "+RNS.prettyhexrep(context_dest)+": "+str(e), RNS.LOG_ERROR)
return False
def requests_allowed_from(self, context_dest):
try:
existing_conv = self._db_conversation(context_dest) existing_conv = self._db_conversation(context_dest)
if existing_conv != None: if existing_conv != None:
if existing_conv["trust"] == 1:
trusted = True
else:
trusted = False
if self.config["telemetry_allow_requests_from_trusted"] == True:
return trusted
cd = existing_conv["data"] cd = existing_conv["data"]
if cd != None and "allow_requests" in cd and cd["allow_requests"] == True: if cd != None and "allow_requests" in cd and cd["allow_requests"] == True:
return True return True
@ -977,15 +981,18 @@ class SidebandCore():
return "not_sent" return "not_sent"
def send_latest_telemetry(self, to_addr=None, stream=None): def send_latest_telemetry(self, to_addr=None, stream=None, is_authorized_telemetry_request=False):
if to_addr == None or to_addr == self.lxmf_destination.hash: if to_addr == None or to_addr == self.lxmf_destination.hash:
return "no_address" return "no_address"
else: else:
if to_addr == self.config["telemetry_collector"]:
is_authorized_telemetry_request = True
if self.getstate(f"telemetry.{RNS.hexrep(to_addr, delimit=False)}.update_sending") == True: if self.getstate(f"telemetry.{RNS.hexrep(to_addr, delimit=False)}.update_sending") == True:
RNS.log("Not sending new telemetry update, since an earlier transfer is already in progress", RNS.LOG_DEBUG) RNS.log("Not sending new telemetry update, since an earlier transfer is already in progress", RNS.LOG_DEBUG)
return "in_progress" return "in_progress"
if self.latest_packed_telemetry != None and self.latest_telemetry != None: if (self.latest_packed_telemetry != None and self.latest_telemetry != None) or stream != None:
dest_identity = RNS.Identity.recall(to_addr) dest_identity = RNS.Identity.recall(to_addr)
if dest_identity == None: if dest_identity == None:
@ -1002,7 +1009,7 @@ class SidebandCore():
else: else:
desired_method = LXMF.LXMessage.DIRECT desired_method = LXMF.LXMessage.DIRECT
lxm_fields = self.get_message_fields(to_addr) lxm_fields = self.get_message_fields(to_addr, is_authorized_telemetry_request=is_authorized_telemetry_request)
if stream != None and len(stream) > 0: if stream != None and len(stream) > 0:
lxm_fields[LXMF.FIELD_TELEMETRY_STREAM] = stream lxm_fields[LXMF.FIELD_TELEMETRY_STREAM] = stream
@ -1041,7 +1048,7 @@ class SidebandCore():
else: else:
RNS.log("A telemetry update was requested, but there was nothing to send.", RNS.LOG_WARNING) RNS.log("A telemetry update was requested, but there was nothing to send.", RNS.LOG_WARNING)
return "not_sent" return "nothing_to_send"
def list_telemetry(self, context_dest = None, after = None, before = None, limit = None): def list_telemetry(self, context_dest = None, after = None, before = None, limit = None):
@ -1559,10 +1566,11 @@ class SidebandCore():
return results return results
def _db_save_telemetry(self, context_dest, telemetry, physical_link = None, source_dest = None): def _db_save_telemetry(self, context_dest, telemetry, physical_link = None, source_dest = None, via = None):
try: try:
remote_telemeter = Telemeter.from_packed(telemetry) remote_telemeter = Telemeter.from_packed(telemetry)
telemetry_timestamp = remote_telemeter.read_all()["time"]["utc"] read_telemetry = remote_telemeter.read_all()
telemetry_timestamp = read_telemetry["time"]["utc"]
db = self.__db_connect() db = self.__db_connect()
dbc = db.cursor() dbc = db.cursor()
@ -1572,7 +1580,8 @@ class SidebandCore():
result = dbc.fetchall() result = dbc.fetchall()
if len(result) != 0: if len(result) != 0:
return RNS.log("Telemetry entry with source "+RNS.prettyhexrep(context_dest)+" and timestamp "+str(telemetry_timestamp)+" already exists, skipping save", RNS.LOG_DEBUG)
return None
if physical_link != None and len(physical_link) != 0: if physical_link != None and len(physical_link) != 0:
remote_telemeter.synthesize("physical_link") remote_telemeter.synthesize("physical_link")
@ -1603,6 +1612,20 @@ class SidebandCore():
remote_telemeter.sensors["received"].update_data() remote_telemeter.sensors["received"].update_data()
telemetry = remote_telemeter.packed() telemetry = remote_telemeter.packed()
if via != None:
if not "received" in remote_telemeter.sensors:
remote_telemeter.synthesize("received")
if "by" in remote_telemeter.sensors["received"].data:
remote_telemeter.sensors["received"].by = remote_telemeter.sensors["received"].data["by"]
if "distance" in remote_telemeter.sensors["received"].data:
remote_telemeter.sensors["received"].geodesic_distance = remote_telemeter.sensors["received"].data["distance"]["geodesic"]
remote_telemeter.sensors["received"].euclidian_distance = remote_telemeter.sensors["received"].data["distance"]["euclidian"]
remote_telemeter.sensors["received"].via = via
remote_telemeter.sensors["received"].update_data()
telemetry = remote_telemeter.packed()
query = "INSERT INTO telemetry (dest_context, ts, data) values (?, ?, ?)" query = "INSERT INTO telemetry (dest_context, ts, data) values (?, ?, ?)"
data = (context_dest, telemetry_timestamp, telemetry) data = (context_dest, telemetry_timestamp, telemetry)
dbc.execute(query, data) dbc.execute(query, data)
@ -1617,6 +1640,13 @@ class SidebandCore():
def _db_update_appearance(self, context_dest, timestamp, appearance): def _db_update_appearance(self, context_dest, timestamp, appearance):
conv = self._db_conversation(context_dest) conv = self._db_conversation(context_dest)
if conv == None:
ae = [appearance, int(time.time())]
# TODO: Clean out these temporary values at some interval.
# Probably expire after 14 days or so.
self.setpersistent("temp.peer_appearance."+RNS.hexrep(context_dest, delimit=False), ae)
else:
data_dict = conv["data"] data_dict = conv["data"]
if data_dict == None: if data_dict == None:
data_dict = {} data_dict = {}
@ -1641,11 +1671,21 @@ class SidebandCore():
if context_dest == self.lxmf_destination.hash: if context_dest == self.lxmf_destination.hash:
return [self.config["telemetry_icon"], self.config["telemetry_fg"], self.config["telemetry_bg"]] return [self.config["telemetry_icon"], self.config["telemetry_fg"], self.config["telemetry_bg"]]
else: else:
if conv == None: data_dict = None
conv = self._db_conversation(context_dest) if conv != None:
if conv != None and "data" in conv:
data_dict = conv["data"] data_dict = conv["data"]
else:
conv = self._db_conversation(context_dest)
if conv != None:
data_dict = conv["data"]
else:
data_dict = {}
apd = self.getpersistent("temp.peer_appearance."+RNS.hexrep(context_dest, delimit=False))
if apd != None:
data_dict["appearance"] = apd
if data_dict != None:
try: try:
if data_dict != None and "appearance" in data_dict: if data_dict != None and "appearance" in data_dict:
def htf(cbytes): def htf(cbytes):
@ -2029,7 +2069,9 @@ class SidebandCore():
if not originator and lxm.fields != None: if not originator and lxm.fields != None:
if self.config["telemetry_receive_trusted_only"] == False or (self.config["telemetry_receive_trusted_only"] == True and self.is_trusted(context_dest)): if self.config["telemetry_receive_trusted_only"] == False or (self.config["telemetry_receive_trusted_only"] == True and self.is_trusted(context_dest)):
if LXMF.FIELD_ICON_APPEARANCE in lxm.fields: if LXMF.FIELD_ICON_APPEARANCE in lxm.fields:
self._db_update_appearance(context_dest, lxm.timestamp, lxm.fields[LXMF.FIELD_ICON_APPEARANCE]) peer_appearance = lxm.fields[LXMF.FIELD_ICON_APPEARANCE]
if peer_appearance != None and len(peer_appearance) > 0 and len(peer_appearance) < 96:
self._db_update_appearance(context_dest, lxm.timestamp, peer_appearance)
if LXMF.FIELD_TELEMETRY in lxm.fields: if LXMF.FIELD_TELEMETRY in lxm.fields:
physical_link = {} physical_link = {}
@ -2040,9 +2082,16 @@ class SidebandCore():
packed_telemetry = self._db_save_telemetry(context_dest, lxm.fields[LXMF.FIELD_TELEMETRY], physical_link=physical_link, source_dest=context_dest) packed_telemetry = self._db_save_telemetry(context_dest, lxm.fields[LXMF.FIELD_TELEMETRY], physical_link=physical_link, source_dest=context_dest)
if LXMF.FIELD_TELEMETRY_STREAM in lxm.fields: if LXMF.FIELD_TELEMETRY_STREAM in lxm.fields:
if lxm.fields[LXMF.FIELD_TELEMETRY_STREAM] != None and len(lxm.fields[LXMF.FIELD_TELEMETRY_STREAM]) > 0:
for telemetry_entry in lxm.fields[LXMF.FIELD_TELEMETRY_STREAM]: for telemetry_entry in lxm.fields[LXMF.FIELD_TELEMETRY_STREAM]:
# TODO: Implement tsource = telemetry_entry[0]
RNS.log("TODO: Save this telemetry stream entry: "+str(telemetry_entry), RNS.LOG_WARNING) ttstemp = telemetry_entry[1]
tpacked = telemetry_entry[2]
if self._db_save_telemetry(tsource, tpacked, via = context_dest):
RNS.log("Saved telemetry stream entry from "+RNS.prettyhexrep(tsource), RNS.LOG_WARNING)
else:
RNS.log("Received telemetry stream field with no data: "+str(lxm.fields[LXMF.FIELD_TELEMETRY_STREAM]), RNS.LOG_DEBUG)
if own_command or len(lxm.content) != 0 or len(lxm.title) != 0: if own_command or len(lxm.content) != 0 or len(lxm.title) != 0:
db = self.__db_connect() db = self.__db_connect()
@ -2482,7 +2531,8 @@ class SidebandCore():
self.pending_telemetry_send = True self.pending_telemetry_send = True
self.pending_telemetry_send_try += 1 self.pending_telemetry_send_try += 1
if self.config["telemetry_send_all_to_collector"]: if self.config["telemetry_send_all_to_collector"]:
self.create_telemetry_collector_response(to_addr=collector_address) last_timebase = (self.getpersistent(f"telemetry.{RNS.hexrep(collector_address, delimit=False)}.last_send_success_timebase") or 0)
self.create_telemetry_collector_response(to_addr=collector_address, timebase=last_timebase, is_authorized_telemetry_request=True)
else: else:
self.send_latest_telemetry(to_addr=collector_address) self.send_latest_telemetry(to_addr=collector_address)
else: else:
@ -2996,9 +3046,9 @@ class SidebandCore():
except Exception as e: except Exception as e:
RNS.log("Error while setting last successul telemetry timebase for "+RNS.prettyhexrep(message.destination_hash), RNS.LOG_DEBUG) RNS.log("Error while setting last successul telemetry timebase for "+RNS.prettyhexrep(message.destination_hash), RNS.LOG_DEBUG)
def get_message_fields(self, context_dest, telemetry_update=False): def get_message_fields(self, context_dest, telemetry_update=False, is_authorized_telemetry_request=False):
fields = {} fields = {}
send_telemetry = (telemetry_update == True) or self.should_send_telemetry(context_dest) send_telemetry = (telemetry_update == True) or (self.should_send_telemetry(context_dest) or is_authorized_telemetry_request)
send_appearance = self.config["telemetry_send_appearance"] or send_telemetry send_appearance = self.config["telemetry_send_appearance"] or send_telemetry
if send_telemetry and self.latest_packed_telemetry != None: if send_telemetry and self.latest_packed_telemetry != None:
@ -3379,7 +3429,7 @@ class SidebandCore():
return return
if message.signature_validated and LXMF.FIELD_COMMANDS in message.fields: if message.signature_validated and LXMF.FIELD_COMMANDS in message.fields:
if self.requests_allowed_from(context_dest): if self.allow_request_from(context_dest):
commands = message.fields[LXMF.FIELD_COMMANDS] commands = message.fields[LXMF.FIELD_COMMANDS]
self.handle_commands(commands, message) self.handle_commands(commands, message)
else: else:
@ -3391,7 +3441,7 @@ class SidebandCore():
self.lxm_ingest(message) self.lxm_ingest(message)
except Exception as e: except Exception as e:
RNS.log("Error while ingesting LXMF message "+RNS.prettyhexrep(message.hash)+" to database: "+str(e)) RNS.log("Error while ingesting LXMF message "+RNS.prettyhexrep(message.hash)+" to database: "+str(e), RNS.LOG_ERROR)
def handle_commands(self, commands, message): def handle_commands(self, commands, message):
try: try:
@ -3403,7 +3453,7 @@ class SidebandCore():
RNS.log("Handling telemetry request with timebase "+str(timebase), RNS.LOG_DEBUG) RNS.log("Handling telemetry request with timebase "+str(timebase), RNS.LOG_DEBUG)
if self.config["telemetry_collector_enabled"]: if self.config["telemetry_collector_enabled"]:
RNS.log(f"Collector requests enabled, returning complete telemetry response for all known objects since {timebase}", RNS.LOG_DEBUG) RNS.log(f"Collector requests enabled, returning complete telemetry response for all known objects since {timebase}", RNS.LOG_DEBUG)
self.create_telemetry_collector_response(to_addr=context_dest, timebase=timebase) self.create_telemetry_collector_response(to_addr=context_dest, timebase=timebase, is_authorized_telemetry_request=True)
else: else:
RNS.log("Responding with own latest telemetry", RNS.LOG_DEBUG) RNS.log("Responding with own latest telemetry", RNS.LOG_DEBUG)
self.send_latest_telemetry(to_addr=context_dest) self.send_latest_telemetry(to_addr=context_dest)
@ -3436,25 +3486,33 @@ class SidebandCore():
except Exception as e: except Exception as e:
RNS.log("Error while handling commands: "+str(e), RNS.LOG_ERROR) RNS.log("Error while handling commands: "+str(e), RNS.LOG_ERROR)
def create_telemetry_collector_response(self, to_addr, timebase): def create_telemetry_collector_response(self, to_addr, timebase, is_authorized_telemetry_request=False):
sources = {} added_sources = {}
sources = self.list_telemetry(after=timebase) sources = self.list_telemetry(after=timebase)
only_latest = self.config["telemetry_requests_only_send_latest"] only_latest = self.config["telemetry_requests_only_send_latest"]
elements = 0; added = 0
telemetry_stream = [] telemetry_stream = []
for source in sources: for source in sources:
if source != to_addr: if source != to_addr:
for entry in sources[source]: for entry in sources[source]:
elements += 1
timestamp = entry[0]; packed_telemetry = entry[1] timestamp = entry[0]; packed_telemetry = entry[1]
te = [source, timestamp, packed_telemetry] te = [source, timestamp, packed_telemetry]
if only_latest: if only_latest:
if not source in sources: if not source in added_sources:
sources[source] = True added_sources[source] = True
telemetry_stream.append(te) telemetry_stream.append(te)
added += 1
else: else:
telemetry_stream.append(te) telemetry_stream.append(te)
added += 1
self.send_latest_telemetry(to_addr=to_addr, stream=telemetry_stream) return self.send_latest_telemetry(
to_addr=to_addr,
stream=telemetry_stream,
is_authorized_telemetry_request=is_authorized_telemetry_request
)
def get_display_name_bytes(self): def get_display_name_bytes(self):

2
sbapp/ui/conversations.py
View File

@ -136,8 +136,6 @@ class Conversations():
icon_color=fg, md_bg_color=bg, icon_color=fg, md_bg_color=bg,
on_release=self.app.conversation_action) on_release=self.app.conversation_action)
RNS.log("ICON "+str(iconl.md_bg_color))
iconl._default_icon_pad = dp(ic_p) iconl._default_icon_pad = dp(ic_p)
iconl.icon_size = dp(ic_s) iconl.icon_size = dp(ic_s)

58
sbapp/ui/objectdetails.py
View File

@ -140,6 +140,12 @@ class ObjectDetails():
pds = self.app.sideband.peer_display_name(source_dest) pds = self.app.sideband.peer_display_name(source_dest)
appearance = self.app.sideband.peer_appearance(source_dest) appearance = self.app.sideband.peer_appearance(source_dest)
self.screen.ids.name_label.text = pds self.screen.ids.name_label.text = pds
if source_dest == own_address:
self.screen.ids.name_label.text = pds+" (this device)"
elif source_dest == self.app.sideband.config["telemetry_collector"]:
self.screen.ids.name_label.text = pds+" (collector)"
self.screen.ids.coordinates_button.disabled = True self.screen.ids.coordinates_button.disabled = True
self.screen.ids.object_appearance.icon = appearance[0] self.screen.ids.object_appearance.icon = appearance[0]
self.screen.ids.object_appearance.icon_color = appearance[1] self.screen.ids.object_appearance.icon_color = appearance[1]
@ -170,8 +176,6 @@ class ObjectDetails():
relative_to = None relative_to = None
if source_dest != own_address: if source_dest != own_address:
relative_to = self.app.sideband.telemeter relative_to = self.app.sideband.telemeter
else:
self.screen.ids.name_label.text = pds+" (this device)"
rendered_telemetry = telemeter.render(relative_to=relative_to) rendered_telemetry = telemeter.render(relative_to=relative_to)
if "location" in telemeter.sensors: if "location" in telemeter.sensors:
@ -291,7 +295,7 @@ class RVDetails(MDRecycleView):
"Battery": 70, "Battery": 70,
"Timestamp": 80, "Timestamp": 80,
"Received": 90, "Received": 90,
"Information": 100, "Information": 5,
} }
self.entries = [] self.entries = []
rendered_telemetry.sort(key=lambda s: sort[s["name"]] if s["name"] in sort else 1000) rendered_telemetry.sort(key=lambda s: sort[s["name"]] if s["name"] in sort else 1000)
@ -323,29 +327,43 @@ class RVDetails(MDRecycleView):
formatted_values = f"[b]Information[/b]: {external_text}" formatted_values = f"[b]Information[/b]: {external_text}"
elif name == "Received": elif name == "Received":
formatted_values = "" formatted_values = ""
by = s["values"]["by"]; by_str = "" by = s["values"]["by"];
if by != None: via = s["values"]["via"];
if by == self.app.sideband.lxmf_destination.hash: if by == self.app.sideband.lxmf_destination.hash:
by_str = "Directly by [b]this device[/b]"
else:
dstr = self.app.sideband.peer_display_name(by)
by_str = f"By [b]{dstr}[/b]"
formatted_values+=by_str
via = s["values"]["via"]; via_str = ""
if via != None:
if via == self.delegate.object_hash: if via == self.delegate.object_hash:
via_str = "directly [b]from emitter[/b]" formatted_values = "Collected directly by [b]this device[/b], directly [b]from emitter[/b]"
else: else:
dstr = self.app.sideband.peer_display_name(by) via_str = self.app.sideband.peer_display_name(via)
via_str = f"via [b]{dstr}[/b]" if via_str == None:
if len(formatted_values) != 0: formatted_values += ", " via_str = "an [b]unknown peer[/b]"
formatted_values += via_str formatted_values = f"Collected directly by [b]this device[/b], via {via_str}"
else:
if via != None and via == by:
vstr = self.app.sideband.peer_display_name(via)
formatted_values = f"Received from and collected by [b]{vstr}[/b]"
if formatted_values != "":
formatted_values = f"Collected {formatted_values}"
else: else:
if via != None:
vstr = self.app.sideband.peer_display_name(via)
via_str = f"Received from [b]{vstr}[/b]"
else:
via_str = "Received from an [b]unknown peer[/b]"
if by != None:
dstr = self.app.sideband.peer_display_name(by)
by_str = f", collected by [b]{dstr}[/b]"
else:
by_str = f", collected by an [b]unknown peer[/b]"
formatted_values = f"{via_str}{by_str}"
if formatted_values == "":
formatted_values = None formatted_values = None
if not by == self.app.sideband.lxmf_destination.hash and not self.app.sideband.is_trusted(by):
extra_entries.append({"icon": "alert", "text": "Collected by a [b]non-trusted[/b] peer"})
elif name == "Battery": elif name == "Battery":
p = s["values"]["percent"] p = s["values"]["percent"]
cs = s["values"]["_meta"] cs = s["values"]["_meta"]