Added audio messaging

sbapp/buildozer.spec

@@ -16,13 +16,13 @@ android.numeric_version = 20240531
 # relevant PRs have now been merged in Kivy/P4A, the next release will hopefully allow
 # building a non-ancient PyCa/Cryptography distribution again. When this happens, add
 # the "cryptography" dependency back in here.
-requirements = kivy==2.3.0,libbz2,pillow==10.2.0,qrcode==7.3.1,usb4a,usbserial4a,libwebp,cryptography
+requirements = kivy==2.3.0,libbz2,pillow==10.2.0,qrcode==7.3.1,usb4a,usbserial4a,libwebp,libogg,libopus,opusfile,numpy,cryptography,pydub,ffpyplayer
 
 android.gradle_dependencies =  com.android.support:support-compat:28.0.0
 #android.enable_androidx = True
 #android.add_aars = patches/support-compat-28.0.0.aar
 
-p4a.local_recipes = ../Others/python-for-android/pythonforandroid/recipes
+p4a.local_recipes = ../recipes/
 
 icon.filename = %(source.dir)s/assets/icon.png
 presplash.filename = %(source.dir)s/assets/presplash_small.png

sbapp/main.py

@@ -73,12 +73,14 @@ if args.daemon:
     NewConv = DaemonElement; Telemetry = DaemonElement; ObjectDetails = DaemonElement; Announces = DaemonElement;
     Messages = DaemonElement; ts_format = DaemonElement; messages_screen_kv = DaemonElement; plyer = DaemonElement; multilingual_markup = DaemonElement;
     ContentNavigationDrawer = DaemonElement; DrawerList = DaemonElement; IconListItem = DaemonElement; escape_markup = DaemonElement;
+    SoundLoader = DaemonElement;
 
 else:
     from kivymd.app import MDApp
     app_superclass = MDApp
     from kivy.core.window import Window
     from kivy.core.clipboard import Clipboard
+    from kivy.core.audio import SoundLoader
     from kivy.base import EventLoop
     from kivy.clock import Clock
     from kivy.lang.builder import Builder
@@ -102,7 +104,7 @@ else:
     import kivy.core.image
     kivy.core.image.Logger = redirect_log()
 
-    if RNS.vendor.platformutils.get_platform() == "android":
+    if RNS.vendor.platformutils.is_android():
         from sideband.core import SidebandCore
         import plyer
 
@@ -228,6 +230,9 @@ class SidebandApp(MDApp):
         self.attach_type = None
         self.attach_dialog = None
         self.rec_dialog = None
+        self.last_msg_audio = None
+        self.msg_sound = None
+        self.audio_msg_mode = LXMF.AM_OPUS_OGG
 
         Window.softinput_mode = "below_target"
         self.icon = self.sideband.asset_dir+"/icon.png"
@@ -1238,7 +1243,7 @@ class SidebandApp(MDApp):
         self.open_conversations(direction="right")
 
     def message_send_action(self, sender=None):
-        if self.messages_view.ids.message_text.text == "":
+        if not (self.attach_type != None and self.attach_path != None) and self.messages_view.ids.message_text.text == "":
             return
 
         def cb(dt):
@@ -1265,10 +1270,14 @@ class SidebandApp(MDApp):
 
             else:
                 msg_content = self.messages_view.ids.message_text.text
+                if msg_content == "":
+                    msg_content = " "
+
                 context_dest = self.messages_view.ids.messages_scrollview.active_conversation
 
                 attachment = None
                 image = None
+                audio = None
                 if not self.outbound_mode_command and not self.outbound_mode_paper:
                     if self.attach_type != None and self.attach_path != None:
                         try:
@@ -1279,6 +1288,11 @@ class SidebandApp(MDApp):
                                 with open(self.attach_path, "rb") as af:
                                     attachment = [fbn, af.read()]
 
+                            if self.attach_type == "audio":
+                                if self.audio_msg_mode == LXMF.AM_OPUS_OGG:
+                                    with open(self.attach_path, "rb") as af:
+                                        audio = [self.audio_msg_mode, af.read()]
+
                             elif self.attach_type == "lbimg":
                                 max_size = 320, 320
                                 with PilImage.open(self.attach_path) as im:
@@ -1350,7 +1364,7 @@ class SidebandApp(MDApp):
                         self.messages_view.ids.messages_scrollview.scroll_y = 0
                         self.jobs(0)
                 
-                elif self.sideband.send_message(msg_content, context_dest, self.outbound_mode_propagation, attachment = attachment, image = image):
+                elif self.sideband.send_message(msg_content, context_dest, self.outbound_mode_propagation, attachment = attachment, image = image, audio = audio):
                     self.messages_view.ids.message_text.text = ""
                     self.messages_view.ids.messages_scrollview.scroll_y = 0
                     self.jobs(0)
@@ -1496,6 +1510,41 @@ class SidebandApp(MDApp):
                 ok_button.bind(on_release=ate_dialog.dismiss)
                 ate_dialog.open()
 
+    def play_audio_field(self, audio_field):
+        if audio_field[0] == LXMF.AM_OPUS_OGG:
+            audio_type = "ogg"
+        else:
+            return False
+        
+        temp_path = self.sideband.rec_cache+"/msg."+audio_type
+
+        if audio_type == "ogg":
+            if self.last_msg_audio != audio_field[1]:
+                self.last_msg_audio = audio_field[1]
+                
+                with open(temp_path, "wb") as af:
+                    af.write(self.last_msg_audio)
+
+                if not RNS.vendor.platformutils.is_android():
+                    self.msg_sound = SoundLoader.load(temp_path)
+
+            if RNS.vendor.platformutils.is_android():
+                if self.msg_sound != None and self.msg_sound._player != None and self.msg_sound._player.isPlaying():
+                    self.msg_sound.stop()
+                else:
+                    from plyer import audio
+                    self.msg_sound = audio
+                    self.msg_sound._file_path = temp_path
+                    self.msg_sound.play()
+
+            else:
+                if self.msg_sound != None and self.msg_sound.state == "play":
+                    self.msg_sound.stop()
+                    return True
+                else:
+                    self.msg_sound.play()
+                    return True
+
     def message_record_audio_action(self):
         ss = int(dp(18))
         if self.rec_dialog == None:
@@ -1506,7 +1555,7 @@ class SidebandApp(MDApp):
                 from sbapp.plyer import audio
 
             self.msg_audio = audio
-            self.msg_audio._file_path = self.sideband.rec_cache+"/msg_rec.aac"
+            self.msg_audio._file_path = self.sideband.rec_cache+"/recording.ogg"
 
             def a_rec_action(sender):
                 if not self.rec_dialog.recording:
@@ -1555,12 +1604,48 @@ class SidebandApp(MDApp):
                     a_rec_action(sender)
                 self.rec_dialog.dismiss()
 
-                self.attach_path = self.msg_audio._file_path
+                try:
-                self.update_message_widgets()
+                    if self.audio_msg_mode == LXMF.AM_OPUS_OGG:
-                toast("Attached \""+str(self.attach_path)+"\"")
+                        self.attach_path = self.msg_audio._file_path
+                        RNS.log("Using unmodified OPUS data in OGG container", RNS.LOG_DEBUG)
+                    else:
+                        ap_start = time.time()
+                        from pydub import AudioSegment
+                        if RNS.vendor.platformutils.is_android():
+                            import pyogg
+                        else:
+                            import sbapp.pyogg as pyogg
 
-                # TODO: Remove
+                        opus_file = pyogg.OpusFile(self.msg_audio._file_path)
-                self.attach_type = "file"
+
+                        audio = AudioSegment(
+                            bytes(opus_file.as_array()),
+                            frame_rate=opus_file.frequency,
+                            sample_width=opus_file.bytes_per_sample, 
+                            channels=opus_file.channels,
+                        )
+                        audio = audio.split_to_mono()[0]
+                        audio = audio.apply_gain(-audio.max_dBFS)
+                        
+                        if self.audio_msg_mode >= LXMF.AM_CODEC2_450PWB and self.audio_msg_mode <= LXMF.AM_CODEC2_3200:
+                            audio = audio.set_frame_rate(8000)
+                            audio = audio.set_sample_width(2)
+                            samples = audio.get_array_of_samples()
+
+                            ap_duration = time.time() - ap_start
+                            RNS.log("Audio processing complete in "+RNS.prettytime(ap_duration)+", samples: "+str(len(samples)), RNS.LOG_DEBUG)
+
+                            export_path = self.sideband.rec_cache+"/recording.raw"
+                            with open(export_path, "wb") as export_file:
+                                export_file.write(samples.tobytes())
+                            self.attach_path = export_path
+                            os.unlink(self.msg_audio._file_path)
+
+                    self.update_message_widgets()
+                    toast("Added recorded audio to message")
+                
+                except Exception as e:
+                    RNS.trace_exception(e)
 
             cancel_button = MDRectangleFlatButton(text="Cancel", font_size=dp(18))
             rec_item = DialogItem(IconLeftWidget(icon="record"), text="[size="+str(ss)+"]Start Recording[/size]", on_release=a_rec_action)
@@ -1597,7 +1682,7 @@ class SidebandApp(MDApp):
 
     def message_attach_action(self, attach_type=None):
         file_attach_types = ["lbimg", "defimg", "hqimg", "file"]
-        rec_attach_types = ["lbaudio", "defaudio", "hqaudio"]
+        rec_attach_types = ["audio"]
         
         self.attach_path = None
         if attach_type in file_attach_types:
@@ -1621,24 +1706,33 @@ class SidebandApp(MDApp):
             def a_file(sender):
                 self.attach_dialog.dismiss()
                 self.message_attach_action(attach_type="file")
+            def a_audio_hq(sender):
+                self.attach_dialog.dismiss()
+                self.audio_msg_mode = LXMF.AM_OPUS_OGG
+                self.message_attach_action(attach_type="audio")
             def a_audio_lb(sender):
                 self.attach_dialog.dismiss()
-                self.message_attach_action(attach_type="lbaudio")
+                self.audio_msg_mode = LXMF.AM_CODEC2_3200
+                self.message_attach_action(attach_type="audio")
 
             if self.attach_dialog == None:
                 ss = int(dp(18))
                 cancel_button = MDRectangleFlatButton(text="Cancel", font_size=dp(18))
+                ad_items = [
+                        DialogItem(IconLeftWidget(icon="message-image-outline"), text="[size="+str(ss)+"]Low-bandwidth Image[/size]", on_release=a_img_lb),
+                        DialogItem(IconLeftWidget(icon="file-image"), text="[size="+str(ss)+"]Medium Image[/size]", on_release=a_img_def),
+                        DialogItem(IconLeftWidget(icon="image-outline"), text="[size="+str(ss)+"]High-res Image[/size]", on_release=a_img_hq),
+                        DialogItem(IconLeftWidget(icon="microphone-message"), text="[size="+str(ss)+"]Audio Recording[/size]", on_release=a_audio_hq),
+                        DialogItem(IconLeftWidget(icon="file-outline"), text="[size="+str(ss)+"]File Attachment[/size]", on_release=a_file)]
+                
+                if RNS.vendor.platformutils.is_linux():
+                    ad_items.pop(3)
+
                 self.attach_dialog = MDDialog(
                     title="Add Attachment",
                     type="simple",
                     text="Select the type of attachment you want to send with this message\n",
-                    items=[
+                    items=ad_items,
-                        DialogItem(IconLeftWidget(icon="message-image-outline"), text="[size="+str(ss)+"]Low-bandwidth Image[/size]", on_release=a_img_lb),
-                        DialogItem(IconLeftWidget(icon="file-image"), text="[size="+str(ss)+"]Medium Image[/size]", on_release=a_img_def),
-                        DialogItem(IconLeftWidget(icon="image-outline"), text="[size="+str(ss)+"]High-res Image[/size]", on_release=a_img_hq),
-                        DialogItem(IconLeftWidget(icon="microphone-message"), text="[size="+str(ss)+"]Audio Recording[/size]", on_release=a_audio_lb),
-                        DialogItem(IconLeftWidget(icon="file-outline"), text="[size="+str(ss)+"]File Attachment[/size]", on_release=a_file),
-                    ],
                     buttons=[ cancel_button ],
                     width_offset=dp(12),
                 )

sbapp/plyer/platforms/android/audio.py

@@ -29,6 +29,7 @@ class AndroidAudio(Audio):
         self._player = None
         self._check_thread = None
         self._finished_callback = None
+        self._format = "opus"
 
     def _check_playback(self):
         while self._player and self._player.isPlaying():
@@ -41,12 +42,24 @@ class AndroidAudio(Audio):
 
     def _start(self):
         self._recorder = MediaRecorder()
-        self._recorder.setAudioSource(AudioSource.DEFAULT)
+        # AAC Format, decent quality
-        self._recorder.setAudioSamplingRate(44100)
+        if self._format == "aac":
-        self._recorder.setAudioEncodingBitRate(128000)
+            self._recorder.setAudioSource(AudioSource.DEFAULT)
-        self._recorder.setAudioChannels(1)
+            self._recorder.setAudioSamplingRate(48000)
-        self._recorder.setOutputFormat(OutputFormat.MPEG_4)
+            self._recorder.setAudioEncodingBitRate(128000)
-        self._recorder.setAudioEncoder(AudioEncoder.AAC)
+            self._recorder.setAudioChannels(1)
+            self._recorder.setOutputFormat(OutputFormat.MPEG_4)
+            self._recorder.setAudioEncoder(AudioEncoder.AAC)
+
+        else:
+            # OPUS
+            self._recorder.setAudioSource(AudioSource.DEFAULT)
+            self._recorder.setAudioSamplingRate(48000)
+            self._recorder.setAudioEncodingBitRate(128000)
+            self._recorder.setAudioChannels(1)
+            self._recorder.setOutputFormat(OutputFormat.OGG)
+            self._recorder.setAudioEncoder(AudioEncoder.OPUS)
+
         self._recorder.setOutputFile(self.file_path)
 
         self._recorder.prepare()

sbapp/sideband/core.py

@@ -3507,6 +3507,8 @@ class SidebandCore():
                 fields[LXMF.FIELD_FILE_ATTACHMENTS] = [attachment]
             if image != None:
                 fields[LXMF.FIELD_IMAGE] = image
+            if audio != None:
+                fields[LXMF.FIELD_AUDIO] = audio
 
             lxm = LXMF.LXMessage(dest, source, content, title="", desired_method=desired_method, fields = fields)
             

sbapp/ui/helpers.py

@@ -13,6 +13,7 @@ def mdc(color, hue=None):
         hue = "400"
     return get_color_from_hex(colors[color][hue])
 
+color_playing = "Amber"
 color_received = "LightGreen"
 color_delivered = "Blue"
 color_paper = "Indigo"
@@ -21,6 +22,8 @@ color_failed = "Red"
 color_unknown = "Gray"
 intensity_msgs_dark = "800"
 intensity_msgs_light = "500"
+intensity_play_dark = "600"
+intensity_play_light = "300"
 
 class ContentNavigationDrawer(Screen):
     pass

sbapp/ui/messages.py

@@ -34,12 +34,12 @@ if RNS.vendor.platformutils.get_platform() == "android":
     import plyer
     from sideband.sense import Telemeter, Commands
     from ui.helpers import ts_format, file_ts_format, mdc
-    from ui.helpers import color_received, color_delivered, color_propagated, color_paper, color_failed, color_unknown, intensity_msgs_dark, intensity_msgs_light
+    from ui.helpers import color_playing, color_received, color_delivered, color_propagated, color_paper, color_failed, color_unknown, intensity_msgs_dark, intensity_msgs_light, intensity_play_dark, intensity_play_light
 else:
     import sbapp.plyer as plyer
     from sbapp.sideband.sense import Telemeter, Commands
     from .helpers import ts_format, file_ts_format, mdc
-    from .helpers import color_received, color_delivered, color_propagated, color_paper, color_failed, color_unknown, intensity_msgs_dark, intensity_msgs_light
+    from .helpers import color_playing, color_received, color_delivered, color_propagated, color_paper, color_failed, color_unknown, intensity_msgs_dark, intensity_msgs_light, intensity_play_dark, intensity_play_light
 
 if RNS.vendor.platformutils.is_darwin():
     from PIL import Image as PilImage
@@ -127,8 +127,10 @@ class Messages():
 
         if self.app.sideband.config["dark_ui"]:
             intensity_msgs = intensity_msgs_dark
+            intensity_play = intensity_play_dark
         else:
             intensity_msgs = intensity_msgs_light
+            intensity_play = intensity_play_light
 
         for w in self.widgets:
             m = w.m
@@ -161,8 +163,11 @@ class Messages():
                     if msg["title"]:
                         titlestr = "[b]Title[/b] "+msg["title"].decode("utf-8")+"\n"
                     w.heading = titlestr+"[b]Sent[/b] "+txstr+"\n[b]State[/b] "+sphrase+prgstr+"                          "
+                    if w.has_audio:
+                        w.heading += f"\n[b]Audio Recording Included[/b]"
                     m["state"] = msg["state"]
 
+
                 if msg["state"] == LXMF.LXMessage.DELIVERED:
                     w.md_bg_color = msg_color = mdc(color_delivered, intensity_msgs)
                     txstr = time.strftime(ts_format, time.localtime(msg["sent"]))
@@ -170,6 +175,8 @@ class Messages():
                     if msg["title"]:
                         titlestr = "[b]Title[/b] "+msg["title"].decode("utf-8")+"\n"
                     w.heading = titlestr+"[b]Sent[/b] "+txstr+"\n[b]State[/b] Delivered"
+                    if w.has_audio:
+                        w.heading += f"\n[b]Audio Recording Included[/b]"
                     m["state"] = msg["state"]
 
                 if msg["method"] == LXMF.LXMessage.PAPER:
@@ -188,6 +195,8 @@ class Messages():
                     if msg["title"]:
                         titlestr = "[b]Title[/b] "+msg["title"].decode("utf-8")+"\n"
                     w.heading = titlestr+"[b]Sent[/b] "+txstr+"\n[b]State[/b] On Propagation Net"
+                    if w.has_audio:
+                        w.heading += f"\n[b]Audio Recording Included[/b]"
                     m["state"] = msg["state"]
 
                 if msg["state"] == LXMF.LXMessage.FAILED:
@@ -198,15 +207,19 @@ class Messages():
                         titlestr = "[b]Title[/b] "+msg["title"].decode("utf-8")+"\n"
                     w.heading = titlestr+"[b]Sent[/b] "+txstr+"\n[b]State[/b] Failed"
                     m["state"] = msg["state"]
+                    if w.has_audio:
+                        w.heading += f"\n[b]Audio Recording Included[/b]"
                     w.dmenu.items.append(w.dmenu.retry_item)
 
 
     def update_widget(self):
         if self.app.sideband.config["dark_ui"]:
             intensity_msgs = intensity_msgs_dark
+            intensity_play = intensity_play_dark
             mt_color = [1.0, 1.0, 1.0, 0.8]
         else:
             intensity_msgs = intensity_msgs_light
+            intensity_play = intensity_play_light
             mt_color = [1.0, 1.0, 1.0, 0.95]
 
         self.ids.message_text.font_name = self.app.input_font
@@ -230,7 +243,9 @@ class Messages():
                 extra_telemetry = {}
                 telemeter = None
                 image_field = None
+                audio_field = None
                 has_image = False
+                has_audio = False
                 attachments_field = None
                 has_attachment = False
                 force_markup = False
@@ -277,6 +292,13 @@ class Messages():
                     except Exception as e:
                         pass
 
+                if "lxm" in m and m["lxm"] and m["lxm"].fields != None and LXMF.FIELD_AUDIO in m["lxm"].fields:
+                    try:
+                        audio_field = m["lxm"].fields[LXMF.FIELD_AUDIO]
+                        has_audio = True
+                    except Exception as e:
+                        pass
+
                 if "lxm" in m and m["lxm"] and m["lxm"].fields != None and LXMF.FIELD_FILE_ATTACHMENTS in m["lxm"].fields:
                     if len(m["lxm"].fields[LXMF.FIELD_FILE_ATTACHMENTS]) > 0:
                         try:
@@ -380,15 +402,36 @@ class Messages():
                         heading_str += str(attachment[0])+", "
                     heading_str = heading_str[:-2]
 
+                if has_audio:
+                    heading_str += f"\n[b]Audio Recording Included[/b]"
+
                 item = ListLXMessageCard(
                     text=pre_content+message_markup.decode("utf-8")+extra_content,
                     heading=heading_str,
                     md_bg_color=msg_color,
                 )
+                item.lsource = m["source"]
 
                 if has_attachment:
                     item.attachments_field = attachments_field
 
+                if has_audio:
+                    def play_audio(sender):
+                        self.app.play_audio_field(sender.audio_field)
+                        stored_color = sender.md_bg_color
+                        if sender.lsource == self.app.sideband.lxmf_destination.hash:
+                            sender.md_bg_color = mdc(color_delivered, intensity_play)
+                        else:
+                            sender.md_bg_color = mdc(color_received, intensity_play)
+
+                        def cb(dt):
+                            sender.md_bg_color = stored_color
+                        Clock.schedule_once(cb, 0.25)
+
+                    item.has_audio = True
+                    item.audio_field = audio_field
+                    item.bind(on_release=play_audio)
+
                 if image_field != None:
                     item.has_image = True
                     item.image_field = image_field

setup.py

@@ -88,7 +88,8 @@ setuptools.setup(
             'sideband=sbapp:main.run',
         ]
     },
-    install_requires=["rns>=0.7.5", "lxmf>=0.4.3", "kivy>=2.3.0", "plyer", "pillow>=10.2.0", "qrcode", "materialyoucolor>=2.0.7"],
+    # TODO: Include pydub
+    install_requires=["rns>=0.7.5", "lxmf>=0.4.3", "kivy>=2.3.0", "pillow>=10.2.0", "qrcode", "materialyoucolor>=2.0.7", "pydub", "ffpyplayer"],
     extras_require={
         "macos": ["pyobjus"],
     },