finished desktop app

app/SpaceOut.json

@@ -0,0 +1,12 @@
+{
+  "type": "service_account",
+  "project_id": "spaceout-298503",
+  "private_key_id": "25dc149b194f8d1f68469477cdfc42b996d55be8",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC4dZI4+InNTpm5\nypnUZPgVthv8gcr1k1mYWQYkH1TTIaL51qxcDBs5CczUh759YUOsS6TRc4CI2Dei\nvti1aTgwhczq/UPVhMuypM25kiu6SAZ0+sZvCkuCjcMEBVKKVehG9BIMq2wGubCD\nc/Iuv7vB8wUmyMKjC4hdWWL1fVYPNdcUnt776H2oCk+mZ6YwqomPPaxtSW+i4AGu\n7YoMxjolg5TN1TUJte/SPg0aAJSsOSI7f6hUf7Jr4pHLxkQnOWFsPw3ezjeWvXZl\nthUYV8DIvEQfE4MYmR2+a6CTpzLR0N04khzLXSRTNx1eB0ueURXjt6PraceQao96\n+yqvdZc9AgMBAAECggEAIirpDHOBNw3trLwKFY0kZQUoFvRFz4pdSLqIyC0jjb5H\nzYaFw8EcU8rsbZu9XcUj/2i9nWyLLQ379EHsq2HTni1SoV6Lb6QbBTrAvrSENAu+\nYnHHSu85wHOY4YhI20YBcg8ovr8MEgzYVOknvaAXW9wzopUCdKgguMXjbjyqscNC\nwoK4K3T04dxVG6wyaXXUqFhd54YmsSyqfaXPEd2pYa8gyqf+e4NmYok+GiO/z+wI\nIyIbcZz3EbCs2BJ85aZKgFPOXKEXRxlwuw7xxdbriTh6uPzIIdU1IegRLddo+KhK\nEfBoEoVCTcATUdQwupIzSrAiwAOaJLs8F0KKA1tWoQKBgQD0uGJp6rBVX5O6OfT1\n2/mPFZjaLBYDFEvnx+Bj89Yoys4wXHpkELeRwTJg+V/uNRdxGF++EGpqWgkK+Feo\nuNjyweGTGAPycvsSf3yKd5tqSz6LY7ceSDk2gvsX+5+29nWMMypoWx+y9ZmwwLby\n6nw63T6nuweh4twGROylyjnR7QKBgQDA9iCreJdKH3aYSqBtYhEQcgJnyGKwS74V\nkqZvKg52EG+PMW4xg6E7z2frcnwFZpdaKoLLxI3kPChyztlTFdLNBaZAPLH1Fb3c\nH4HTJ5nh5ALPcc4kFNSK7/X06k2K7vlIhIvW1bWYseguL1lsekiG8vT+18uINJj3\nxB1ROJZwkQKBgGrLbGc8e+dF5noGgNgqPyYqDqJnStPdL6LenxX/ex4iIwkH0oGI\nqhN5dDrNmQejM6+vK1kOYOI4mGmpJtgCkuqdoYtHl7FebCMOb5Mdzzz7yTebNHaK\nni0jy+ATdwepVnLwgTk5SwQWGhQAhdZMbhpiIs2f2RzUm6BAw+U18zWhAoGBAKJ8\n4EfkdWmqkwBtHyjdAseZaeMg/9G7Bmc+Jb7IaIMNFhQ7qLIzSMuHvNesgTk/CcaY\ns6mJa369FcaP3ruzTd7tmfDP638ZftZlBbrcxx1MFv2+tLr3e38/0BscTo3m7K4f\nR25yacgaUAzMPH43ful8n8gVycN5nzJMx+9EOpKxAoGBAM6xhmm/z4FEYHPARWCl\nKfYZ0M/UBtsCQ91E5XroijQd/1Q49FbY5iswZHLCcwifW/+FMzQpSV0Q8zj+t5RS\n618PJxx96Ke+gNjX88qRlag2pdhjZacIe6e1berUgANW3szc0YlMFyu+I8PqtqME\nvZOX9DIJStyMPA1M7Rh1zTgY\n-----END PRIVATE KEY-----\n",
+  "client_email": "spaceout@spaceout-298503.iam.gserviceaccount.com",
+  "client_id": "114517995111610871266",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/spaceout%40spaceout-298503.iam.gserviceaccount.com"
+}

app/listener.ui

@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>Form</class>
+ <widget class="QWidget" name="Form">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>340</width>
+    <height>202</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>Form</string>
+  </property>
+  <widget class="QWidget" name="horizontalLayoutWidget">
+   <property name="geometry">
+    <rect>
+     <x>30</x>
+     <y>10</y>
+     <width>271</width>
+     <height>121</height>
+    </rect>
+   </property>
+   <layout class="QHBoxLayout" name="horizontalLayout">
+    <item>
+     <widget class="QComboBox" name="comboBox"/>
+    </item>
+   </layout>
+  </widget>
+  <widget class="QPushButton" name="pushButton">
+   <property name="geometry">
+    <rect>
+     <x>50</x>
+     <y>150</y>
+     <width>241</width>
+     <height>31</height>
+    </rect>
+   </property>
+   <property name="text">
+    <string>Start Listening</string>
+   </property>
+  </widget>
+ </widget>
+ <resources/>
+ <connections/>
+</ui>

app/spaceout.py

@@ -11,15 +11,111 @@
 from PyQt5 import QtCore, QtGui, QtWidgets
 from PyQt5.QtWidgets import QDialog, QPushButton, QVBoxLayout, QApplication, QSplashScreen 
 
+from multiprocessing import Process, Queue
+
+import re
+import sys
 import webbrowser
 import requests
 import keyring
+import datetime
+import text_to_speech
 from requests.auth import HTTPBasicAuth
 
 API_ENDPOINT = 'http://localhost:8000/api/'
 
+def on_exit(username, password, request):
+    with open('current_transcript', 'r') as f:
+        first_line = f.readline()
+    print('FIRST LINE: ' + first_line)
+    print("SAM")
+    transcript = open(f'{first_line}.txt', w)
+    transcript.write(open('current_transcript', 'r').read())
+    transcript.close()
+
+def main():
+    
+    app = QtWidgets.QApplication(sys.argv)
+
+    try:
+        f = open('.userinfo')
+        username = f.read()
+        f.close()
+        isLoggedIn = True
+
+    except:
+        isLoggedIn = False
+
+    if not isLoggedIn:
+        Form = QtWidgets.QWidget()
+        ui = LoginForm()
+        ui.setupUI(Form)
+        Form.show()
+
+        sys.exit(app.exec_())
+    else:
+        password = keyring.get_password('spaceout', username)
+        r = requests.get(f'{API_ENDPOINT}profile', auth=HTTPBasicAuth(username, password)) 
+        if not r.ok:
+            raise Exception
+        
+        request = r.json()
+
+        
+        import atexit
+        atexit.register(on_exit, username, password, request)
+
+        ListenerWidget = QtWidgets.QWidget()
+        ui = Listener()
+        ui.setupUI(ListenerWidget, request['classes'], request['user']['first_name'])
+        ListenerWidget.show()
+
+        sys.exit(app.exec_())
+
+
+class Listener(object):
+    def setupUI(self, Form, classes, name):
+        Form.setObjectName("Form")
+        Form.resize(340, 202)
+        self.name = name
+        self.horizontalLayoutWidget = QtWidgets.QWidget(Form)
+        self.horizontalLayoutWidget.setGeometry(QtCore.QRect(30, 10, 271, 121))
+        self.horizontalLayoutWidget.setObjectName("horizontalLayoutWidget")
+        self.horizontalLayout = QtWidgets.QHBoxLayout(self.horizontalLayoutWidget)
+        self.horizontalLayout.setContentsMargins(0, 0, 0, 0)
+        self.horizontalLayout.setObjectName("horizontalLayout")
+        self.comboBox = QtWidgets.QComboBox(self.horizontalLayoutWidget)
+        self.comboBox.setObjectName("comboBox")
+        self.horizontalLayout.addWidget(self.comboBox)
+        self.play_pause = QtWidgets.QPushButton(Form)
+        self.play_pause.setGeometry(QtCore.QRect(50, 150, 241, 31))
+        self.play_pause.setObjectName("play_pause")
+        self.play_pause.clicked.connect(self.start_listener)
+
+        rooms = [f'Period {room["period"]} - {room["name"]} with {room["teacher"]}    ID:({room["id"]})' for room in classes]
+        self.comboBox.addItems(rooms)
+
+        self.retranslateUI(Form)
+        QtCore.QMetaObject.connectSlotsByName(Form)
+
+    def retranslateUI(self, Form):
+        _translate = QtCore.QCoreApplication.translate
+        Form.setWindowTitle(_translate("SpaceOut", "SpaceOut"))
+        self.play_pause.setText(_translate("Form", "Start Listening"))
+
+    
+    def start_listener(self):
+        choice = self.comboBox.currentText()
+        print(choice)
+        result = re.search('ID:(.*)', choice)
+        class_id = result.group(1)[1]
+        transcript = open('current_transcript', 'w')
+        transcript.write(f'{choice}-{str(datetime.datetime.now())}\n')
+        transcript.close()
+        text_to_speech.text_to_speech(self.name)
+
 class LoginForm(object):
-    def setupUi(self, Form):
+    def setupUI(self, Form):
         Form.setObjectName("SpaceOut Login")
         Form.resize(500, 756)
         self.verticalLayout = QtWidgets.QVBoxLayout(Form)
@@ -143,7 +239,7 @@ class LoginForm(object):
         self.horizontalLayout_3.addWidget(self.widget)
         self.verticalLayout.addLayout(self.horizontalLayout_3)
 
-        self.retranslateUi(Form)
+        self.retranslateUI(Form)
         QtCore.QMetaObject.connectSlotsByName(Form)
 
     def open_register_link(self):
@@ -159,15 +255,15 @@ class LoginForm(object):
             f = open('.userinfo', 'w')
             f.write(self.username.text())
             f.close()
-            app.quit()
+            main()
         else:
             msg = QtWidgets.QMessageBox()
             msg.setText('Incorrect Login')
             msg.exec_()
 
-    def retranslateUi(self, Form):
+    def retranslateUI(self, Form):
         _translate = QtCore.QCoreApplication.translate
-        Form.setWindowTitle(_translate("Form", "Form"))
+        Form.setWindowTitle(_translate("SpaceOut", "SpaceOut"))
         self.label.setText(_translate("Form", "<html><head/><body><p><img src=\":/src/spaceout.png\"/></p></body></html>"))
         self.label_2.setText(_translate("Form", "<html><head/><body><p><img src=\":/src/user.png\"/></p></body></html>"))
         self.label_3.setText(_translate("Form", "<html><head/><body><p><img src=\":/src/password.png\"/></p></body></html>"))
@@ -176,23 +272,7 @@ class LoginForm(object):
 
 import spaceout_rc
 
+
+
 if __name__ == "__main__":
-    import sys
+    main()
-    app = QtWidgets.QApplication(sys.argv)
-    try:
-        f = open('.userinfo')
-        username = f.read()
-        f.close()
-        password = keyring.get_password('spaceout', username)
-        r = requests.get(f'{API_ENDPOINT}profile', auth=HTTPBasicAuth(username, password)) 
-        if not r.ok:
-            raise Exception
-
-    except:
-        Form = QtWidgets.QWidget()
-        ui = LoginForm()
-        ui.setupUi(Form)
-        Form.show()
-
-
-    sys.exit(app.exec_())

app/text_to_speech.py

@@ -0,0 +1,252 @@
+import re
+import os
+import sys
+import time
+import winsound
+import webbrowser
+import threading
+from threading import Thread
+
+from pywinauto import application
+from pywinauto.findwindows import WindowAmbiguousError, WindowNotFoundError
+
+from google.cloud import speech
+import pyaudio
+from six.moves import queue
+
+# Audio recording parameters
+STREAMING_LIMIT = 240000
+SAMPLE_RATE = 16000
+CHUNK_SIZE = int(SAMPLE_RATE / 10)
+
+
+def get_current_time():
+    return int(round(time.time() * 1000))
+
+
+class ResumableMicrophoneStream:
+
+    def __init__(self, rate, chunk_size):
+        self._rate = rate
+        self.chunk_size = chunk_size
+        self._num_channels = 1
+        self._buff = queue.Queue()
+        self.closed = True
+        self.start_time = get_current_time()
+        self.restart_counter = 0
+        self.audio_input = []
+        self.last_audio_input = []
+        self.result_end_time = 0
+        self.is_final_end_time = 0
+        self.final_request_end_time = 0
+        self.bridging_offset = 0
+        self.last_transcript_was_final = False
+        self.new_stream = True
+        self._audio_interface = pyaudio.PyAudio()
+        self._audio_stream = self._audio_interface.open(
+            format=pyaudio.paInt16,
+            channels=self._num_channels,
+            rate=self._rate,
+            input=True,
+            frames_per_buffer=self.chunk_size,
+            stream_callback=self._fill_buffer,
+        )
+
+    def __enter__(self):
+
+        self.closed = False
+        return self
+
+    def __exit__(self, type, value, traceback):
+
+        self._audio_stream.stop_stream()
+        self._audio_stream.close()
+        self.closed = True
+        self._buff.put(None)
+        self._audio_interface.terminate()
+
+    def _fill_buffer(self, in_data, *args, **kwargs):
+
+        self._buff.put(in_data)
+        return None, pyaudio.paContinue
+
+    def generator(self):
+
+        while not self.closed:
+            data = []
+
+            if self.new_stream and self.last_audio_input:
+
+                chunk_time = STREAMING_LIMIT / len(self.last_audio_input)
+
+                if chunk_time != 0:
+
+                    if self.bridging_offset < 0:
+                        self.bridging_offset = 0
+
+                    if self.bridging_offset > self.final_request_end_time:
+                        self.bridging_offset = self.final_request_end_time
+
+                    chunks_from_ms = round(
+                        (self.final_request_end_time - self.bridging_offset)
+                        / chunk_time
+                    )
+
+                    self.bridging_offset = round(
+                        (len(self.last_audio_input) - chunks_from_ms) * chunk_time
+                    )
+
+                    for i in range(chunks_from_ms, len(self.last_audio_input)):
+                        data.append(self.last_audio_input[i])
+
+                self.new_stream = False
+
+            chunk = self._buff.get()
+            self.audio_input.append(chunk)
+
+            if chunk is None:
+                return
+            data.append(chunk)
+            # Now consume whatever other data's still buffered.
+            while True:
+                try:
+                    chunk = self._buff.get(block=False)
+
+                    if chunk is None:
+                        return
+                    data.append(chunk)
+                    self.audio_input.append(chunk)
+
+                except queue.Empty:
+                    break
+
+            yield b"".join(data)
+
+
+def listen_print_loop(responses, stream, name):
+    for response in responses:
+
+        if get_current_time() - stream.start_time > STREAMING_LIMIT:
+            stream.start_time = get_current_time()
+            break
+
+        if not response.results:
+            continue
+
+        result = response.results[0]
+
+        if not result.alternatives:
+            continue
+
+        transcript = result.alternatives[0].transcript
+
+        if result.is_final:
+            sys.stdout.write("Final: " + transcript + "\n")
+
+            stream.is_final_end_time = stream.result_end_time
+            stream.last_transcript_was_final = True
+            transcript_file = open("current_transcript", "a")
+            transcript_file.write(f'{time.strftime("%H:%M:%S")} {transcript}\n')
+            transcript_file.close()
+
+            if re.search(r"\b("+ name + r")\b", transcript, re.IGNORECASE):
+                name_called()
+                stream.closed = True
+                break
+
+        else:
+            sys.stdout.write("Speaking: " + transcript + "\r")
+
+            stream.last_transcript_was_final = False
+
+
+def name_called():
+    for i in range(2):
+        winsound.Beep(1500, 250)
+        winsound.Beep(600, 250)
+
+    app = application.Application()
+    try:
+        app.connect(title_re=".*Chrome.*")
+
+        app_dialog = app.top_window()
+        app_dialog.restore()
+        app_dialog.maximize()
+
+    except(WindowNotFoundError):
+        print ("Couldn't open chrome")
+        pass
+    except(WindowAmbiguousError):
+        print ('There are too many Chrome windows found')
+        pass
+
+    transcript = open('current_transcript', 'r')
+    last_spoken = ''
+
+    for line in (transcript.readlines()[-3:]): 
+        last_spoken += line
+
+    print(last_spoken)
+    message_box(last_spoken)
+
+
+
+def message_box (text):
+    from PyQt5 import QtWidgets
+    msg = QtWidgets.QMessageBox()
+    msg.setText(text)
+    msg.show()
+    sys.exit(msg.exec_())
+
+def text_to_speech(name):
+    
+
+    """start bidirectional streaming from microphone input to speech API"""
+    os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = r"SpaceOut.json"
+    
+
+    client = speech.SpeechClient()
+    config = speech.RecognitionConfig(
+        encoding=speech.RecognitionConfig.AudioEncoding.LINEAR16,
+        sample_rate_hertz=SAMPLE_RATE,
+        language_code="en-US",
+        max_alternatives=1,
+    )
+
+    streaming_config = speech.StreamingRecognitionConfig(
+        config=config, interim_results=True
+    )
+
+    mic_manager = ResumableMicrophoneStream(SAMPLE_RATE, CHUNK_SIZE)
+
+    with mic_manager as stream:
+
+        while not stream.closed:
+
+            stream.audio_input = []
+            audio_generator = stream.generator()
+
+            requests = (
+                speech.StreamingRecognizeRequest(audio_content=content)
+                for content in audio_generator
+            )
+
+            responses = client.streaming_recognize(streaming_config, requests)
+
+            # Now, put the transcription responses to use.
+            listen_print_loop(responses, stream, name)
+
+            if stream.result_end_time > 0:
+                stream.final_request_end_time = stream.is_final_end_time
+            stream.result_end_time = 0
+            stream.last_audio_input = []
+            stream.last_audio_input = stream.audio_input
+            stream.audio_input = []
+            stream.restart_counter = stream.restart_counter + 1
+
+            stream.new_stream = True
+        
+        listen_print_loop(responses, stream, name)
+
+if __name__ == "__main__":
+    text_to_speech("lucas") # just for testing if script is directly invoked