mount/luna-frontend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixed some error frontend stuff

Browse Source
This commit is contained in:
Andreas Fruhwirt 2025-04-02 22:17:05 +02:00
parent 29142e3b04
commit 4dd0d91de2
4 changed files with 86 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,2 +1,3 @@
__pycache__ __pycache__
.env .env
venv

50
application.py
View File

@ -42,12 +42,15 @@ def handle_disconnect(client : Client):
info(f"Client disconnected: {client.id}") info(f"Client disconnected: {client.id}")
if client.id in client_to_session_id: if client.id in client_to_session_id:
session_id = client_to_session_id[client.id] session_id = client_to_session_id[client.id]
if session_id in session_to_websocket: cnt = 0
for cid in client_to_session_id:
if client_to_session_id[cid] == session_id:
cnt = cnt + 1
if session_id in session_to_websocket and cnt <= 1:
info(f"Deleting session_id from session_to_websocket")
del session_to_websocket[session_id] del session_to_websocket[session_id]
del client_to_session_id[client.id] del client_to_session_id[client.id]
app.on_disconnect(handle_disconnect)
def start_recording(ui_elements): def start_recording(ui_elements):
if not refresh_ui_enabled(ui_elements): if not refresh_ui_enabled(ui_elements):
return return
@ -105,7 +108,12 @@ def enable_ui(ui_elements, state = 0):
def after_end_audio(ui_elements, output): def after_end_audio(ui_elements, output):
debug("after_end_audio") debug("after_end_audio")
if output["status"] != "ok": if output["status"] == "end_audio_failure":
ui_elements["websocket"].generating = False
refresh_ui_enabled(ui_elements)
with ui_elements["main"]:
ui.notify("Luna didn\'t understand that correctly, please try again!", close_button='OK')
elif output["status"] != "ok":
refresh_ui_enabled(ui_elements) refresh_ui_enabled(ui_elements)
return return
@ -150,6 +158,10 @@ def after_handle_upstream(ui_elements, response):
with ui_elements["main"]: with ui_elements["main"]:
ui_elements["audio"].set_source(f"data:audio/webm;base64,{response['audio']}") ui_elements["audio"].set_source(f"data:audio/webm;base64,{response['audio']}")
start_playback(ui_elements["id"]) start_playback(ui_elements["id"])
if "show" in response:
with ui_elements["main"]:
ui_elements["markdown"].set_content(response["show"])
ui_elements["markdown"].update()
pass pass
@app.post('/api/v1/upload') @app.post('/api/v1/upload')
@ -158,7 +170,10 @@ async def handle_audio(credentials: Annotated[HTTPBasicCredentials, Depends(secu
if not is_authorized(credentials): if not is_authorized(credentials):
return Response({"status": "Unauthorized"}, status_code=401) return Response({"status": "Unauthorized"}, status_code=401)
try: try:
debug(session_to_websocket)
debug(client_to_session_id)
session_id = request.session["id"] session_id = request.session["id"]
debug("session_id = " + session_id)
if session_id not in session_to_websocket: if session_id not in session_to_websocket:
return {"status": "nok"} return {"status": "nok"}
websocket = session_to_websocket[session_id] websocket = session_to_websocket[session_id]
@ -184,7 +199,6 @@ async def main_page(credentials: Annotated[HTTPBasicCredentials, Depends(securit
global main_containers, mic_buttons, text_containers global main_containers, mic_buttons, text_containers
if not is_authorized(credentials): if not is_authorized(credentials):
return Response("Get outta here män :(", status_code=401) return Response("Get outta here män :(", status_code=401)
#await ui.context.client.connected()
session_id = request.session["id"] session_id = request.session["id"]
ui.add_head_html(f"<script type='text/javascript' src='/static/record.js'></script>") ui.add_head_html(f"<script type='text/javascript' src='/static/record.js'></script>")
ui.add_head_html(f"<link type='text/tailwindcss' rel='stylesheet' href='/static/style.css'>") ui.add_head_html(f"<link type='text/tailwindcss' rel='stylesheet' href='/static/style.css'>")
@ -204,13 +218,27 @@ async def main_page(credentials: Annotated[HTTPBasicCredentials, Depends(securit
button.on('touchstart', lambda: start_recording(ui_elements)) button.on('touchstart', lambda: start_recording(ui_elements))
button.on('touchend', lambda: stop_recording(ui_elements)) button.on('touchend', lambda: stop_recording(ui_elements))
ui_elements["micbutton"] = button ui_elements["micbutton"] = button
ui_elements["markdown"] = ui.markdown().props("id=markdown")
audio = ui.audio(src="").classes("disabled opacity-0").props("id=audioout") audio = ui.audio(src="").props("id=audioout")
audio.on('ended', lambda: stop_playback(ui_elements)) audio.on('ended', lambda: stop_playback(ui_elements))
ui_elements["audio"] = audio ui_elements["audio"] = audio
ui_elements["websocket"] = WebSocketClient(ui_elements, after_handle_upstream) if (session_id not in session_to_websocket) or (session_id in session_to_websocket and session_to_websocket[session_id].enabled is False):
ui_elements["websocket"].start_thread(after_websocket_init) if session_id in session_to_websocket and session_to_websocket[session_id].enabled is False:
client_to_session_id[client.id] = session_id del session_to_websocket[session_id]
session_to_websocket[session_id] = ui_elements["websocket"] ui_elements["websocket"] = WebSocketClient(ui_elements, after_handle_upstream)
ui.run(show=False) ui_elements["websocket"].start_thread(after_websocket_init)
debug("assigned client_to_session_id = " + client.id + ", " + session_id)
client_to_session_id[client.id] = session_id
session_to_websocket[session_id] = ui_elements["websocket"]
else:
if client.id in client_to_session_id:
del client_to_session_id[client.id]
client_to_session_id[client.id] = session_id
ui_elements["websocket"] = session_to_websocket[session_id]
session_to_websocket[session_id].ui_elements = ui_elements
refresh_ui_enabled(ui_elements)
ui.context.client.on_disconnect(handle_disconnect)
ui.run(show=False, port=8642)

73
static/record.js
View File

@ -6,6 +6,11 @@ let dataArray;
let animationId; let animationId;
let audioOutSource; let audioOutSource;
console.log("Asking for Audio permissions...");
window.addEventListener("load", async (event) => {
await navigator.mediaDevices.getUserMedia({ audio: true });
});
function startButtonPulse(analyser) { function startButtonPulse(analyser) {
const button = document.querySelector('#recordbutton'); const button = document.querySelector('#recordbutton');
@ -52,6 +57,7 @@ const startPlayback = async () => {
analyser.fftSize = 2048; analyser.fftSize = 2048;
dataArray = new Uint8Array(analyser.frequencyBinCount); dataArray = new Uint8Array(analyser.frequencyBinCount);
audioOutSource.connect(analyser); audioOutSource.connect(analyser);
analyser.connect(audioContext.destination);
} }
startButtonPulse(analyser); startButtonPulse(analyser);
@ -73,41 +79,46 @@ const stopPlayback = async () => {
const startRecording = async () => { const startRecording = async () => {
try { try {
console.log("Started recording!"); navigator.permissions.query({ name: "microphone" }).then(async (result) => {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); if (result.state === "granted") {
mediaRecorder = new MediaRecorder(stream); const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
audioChunks = []; mediaRecorder = new MediaRecorder(stream);
audioChunks = [];
console.log("Started recording!");
// Audio analysis // Audio analysis
audioContext = new AudioContext(); audioContext = new AudioContext();
analyser = audioContext.createAnalyser(); analyser = audioContext.createAnalyser();
const source = audioContext.createMediaStreamSource(stream); const source = audioContext.createMediaStreamSource(stream);
analyser.fftSize = 2048; analyser.fftSize = 2048;
dataArray = new Uint8Array(analyser.frequencyBinCount); dataArray = new Uint8Array(analyser.frequencyBinCount);
source.connect(analyser); source.connect(analyser);
startButtonPulse(analyser); startButtonPulse(analyser);
mediaRecorder.ondataavailable = async event => { mediaRecorder.ondataavailable = async event => {
if (event.data.size > 0) { if (event.data.size > 0) {
audioChunks.push(event.data); console.log("audio available!");
url = "/api/v1/upload"; audioChunks.push(event.data);
if (mediaRecorder.state === 'inactive') { url = "/api/v1/upload";
url = "/api/v1/send"; if (mediaRecorder.state === 'inactive') {
} url = "/api/v1/send";
fetch(url, { }
method: 'POST', fetch(url, {
body: event.data, method: 'POST',
}); body: event.data,
});
}
};
mediaRecorder.onstop = async () => {
console.log("Stopped recording!");
stopButtonPulse();
};
mediaRecorder.start(1000);
} }
}; });
mediaRecorder.onstop = async () => {
console.log("Stopped recording!");
stopButtonPulse();
};
mediaRecorder.start(1000);
} catch (err) { } catch (err) {
console.error('Microphone access denied or error:', err); console.error('Microphone access denied or error:', err);
alert('Please allow microphone access to record audio.'); alert('Please allow microphone access to record audio.');

6
websocket_client.py
View File

@ -1,7 +1,6 @@
from asyncio import Queue from asyncio import Queue
from asyncio.exceptions import TimeoutError from asyncio.exceptions import TimeoutError
import traceback import traceback
import websocket
import os import os
import json import json
import threading import threading
@ -27,6 +26,7 @@ class WebSocketClient():
self._handlers = {} self._handlers = {}
self._audio_source = None self._audio_source = None
self._after_handle_upstream = after_handle_upstream self._after_handle_upstream = after_handle_upstream
self.main_thread = None
def __del__(self): def __del__(self):
if hasattr(self, "_websocket"): if hasattr(self, "_websocket"):
@ -79,6 +79,8 @@ class WebSocketClient():
self.generating = False self.generating = False
self._after_handle_upstream(self.ui_elements, {"audio": b64encode(self._audio_source).decode()}) self._after_handle_upstream(self.ui_elements, {"audio": b64encode(self._audio_source).decode()})
self._audio_source = None self._audio_source = None
elif request["method"] == "show":
self._after_handle_upstream(self.ui_elements, {"show": request["show"]})
async def input_loop(self): async def input_loop(self):
try: try:
@ -156,7 +158,7 @@ class WebSocketClient():
# start thread # start thread
def start_thread(self, after_init_func): def start_thread(self, after_init_func):
threading.Thread(target=self.main_loop, args=(after_init_func,), daemon=True).start() self.main_thread = threading.Thread(target=self.main_loop, args=(after_init_func,), daemon=True).start()
async def check_health(self): async def check_health(self):
try: try: