diff --git a/app/.gitkeep b/app/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/app/SpaceOut.json b/app/SpaceOut.json new file mode 100644 index 00000000..342e900b --- /dev/null +++ b/app/SpaceOut.json @@ -0,0 +1,12 @@ +{ + "type": "service_account", + "project_id": "spaceout-298503", + "private_key_id": "25dc149b194f8d1f68469477cdfc42b996d55be8", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC4dZI4+InNTpm5\nypnUZPgVthv8gcr1k1mYWQYkH1TTIaL51qxcDBs5CczUh759YUOsS6TRc4CI2Dei\nvti1aTgwhczq/UPVhMuypM25kiu6SAZ0+sZvCkuCjcMEBVKKVehG9BIMq2wGubCD\nc/Iuv7vB8wUmyMKjC4hdWWL1fVYPNdcUnt776H2oCk+mZ6YwqomPPaxtSW+i4AGu\n7YoMxjolg5TN1TUJte/SPg0aAJSsOSI7f6hUf7Jr4pHLxkQnOWFsPw3ezjeWvXZl\nthUYV8DIvEQfE4MYmR2+a6CTpzLR0N04khzLXSRTNx1eB0ueURXjt6PraceQao96\n+yqvdZc9AgMBAAECggEAIirpDHOBNw3trLwKFY0kZQUoFvRFz4pdSLqIyC0jjb5H\nzYaFw8EcU8rsbZu9XcUj/2i9nWyLLQ379EHsq2HTni1SoV6Lb6QbBTrAvrSENAu+\nYnHHSu85wHOY4YhI20YBcg8ovr8MEgzYVOknvaAXW9wzopUCdKgguMXjbjyqscNC\nwoK4K3T04dxVG6wyaXXUqFhd54YmsSyqfaXPEd2pYa8gyqf+e4NmYok+GiO/z+wI\nIyIbcZz3EbCs2BJ85aZKgFPOXKEXRxlwuw7xxdbriTh6uPzIIdU1IegRLddo+KhK\nEfBoEoVCTcATUdQwupIzSrAiwAOaJLs8F0KKA1tWoQKBgQD0uGJp6rBVX5O6OfT1\n2/mPFZjaLBYDFEvnx+Bj89Yoys4wXHpkELeRwTJg+V/uNRdxGF++EGpqWgkK+Feo\nuNjyweGTGAPycvsSf3yKd5tqSz6LY7ceSDk2gvsX+5+29nWMMypoWx+y9ZmwwLby\n6nw63T6nuweh4twGROylyjnR7QKBgQDA9iCreJdKH3aYSqBtYhEQcgJnyGKwS74V\nkqZvKg52EG+PMW4xg6E7z2frcnwFZpdaKoLLxI3kPChyztlTFdLNBaZAPLH1Fb3c\nH4HTJ5nh5ALPcc4kFNSK7/X06k2K7vlIhIvW1bWYseguL1lsekiG8vT+18uINJj3\nxB1ROJZwkQKBgGrLbGc8e+dF5noGgNgqPyYqDqJnStPdL6LenxX/ex4iIwkH0oGI\nqhN5dDrNmQejM6+vK1kOYOI4mGmpJtgCkuqdoYtHl7FebCMOb5Mdzzz7yTebNHaK\nni0jy+ATdwepVnLwgTk5SwQWGhQAhdZMbhpiIs2f2RzUm6BAw+U18zWhAoGBAKJ8\n4EfkdWmqkwBtHyjdAseZaeMg/9G7Bmc+Jb7IaIMNFhQ7qLIzSMuHvNesgTk/CcaY\ns6mJa369FcaP3ruzTd7tmfDP638ZftZlBbrcxx1MFv2+tLr3e38/0BscTo3m7K4f\nR25yacgaUAzMPH43ful8n8gVycN5nzJMx+9EOpKxAoGBAM6xhmm/z4FEYHPARWCl\nKfYZ0M/UBtsCQ91E5XroijQd/1Q49FbY5iswZHLCcwifW/+FMzQpSV0Q8zj+t5RS\n618PJxx96Ke+gNjX88qRlag2pdhjZacIe6e1berUgANW3szc0YlMFyu+I8PqtqME\nvZOX9DIJStyMPA1M7Rh1zTgY\n-----END PRIVATE KEY-----\n", + "client_email": "spaceout@spaceout-298503.iam.gserviceaccount.com", + "client_id": "114517995111610871266", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/spaceout%40spaceout-298503.iam.gserviceaccount.com" +} diff --git a/app/listener.ui b/app/listener.ui new file mode 100644 index 00000000..0bc4372c --- /dev/null +++ b/app/listener.ui @@ -0,0 +1,47 @@ + + + Form + + + + 0 + 0 + 340 + 202 + + + + Form + + + + + 30 + 10 + 271 + 121 + + + + + + + + + + + + 50 + 150 + 241 + 31 + + + + Start Listening + + + + + + diff --git a/app/requirements.txt b/app/requirements.txt new file mode 100644 index 00000000..effbe9b1 Binary files /dev/null and b/app/requirements.txt differ diff --git a/app/spaceout.py b/app/spaceout.py index 58140333..896fdad9 100755 --- a/app/spaceout.py +++ b/app/spaceout.py @@ -11,15 +11,111 @@ from PyQt5 import QtCore, QtGui, QtWidgets from PyQt5.QtWidgets import QDialog, QPushButton, QVBoxLayout, QApplication, QSplashScreen +from multiprocessing import Process, Queue + +import re +import sys import webbrowser import requests import keyring +import datetime +import text_to_speech from requests.auth import HTTPBasicAuth API_ENDPOINT = 'http://localhost:8000/api/' +def on_exit(username, password, request): + with open('current_transcript', 'r') as f: + first_line = f.readline() + print('FIRST LINE: ' + first_line) + print("SAM") + transcript = open(f'{first_line}.txt', w) + transcript.write(open('current_transcript', 'r').read()) + transcript.close() + +def main(): + + app = QtWidgets.QApplication(sys.argv) + + try: + f = open('.userinfo') + username = f.read() + f.close() + isLoggedIn = True + + except: + isLoggedIn = False + + if not isLoggedIn: + Form = QtWidgets.QWidget() + ui = LoginForm() + ui.setupUI(Form) + Form.show() + + sys.exit(app.exec_()) + else: + password = keyring.get_password('spaceout', username) + r = requests.get(f'{API_ENDPOINT}profile', auth=HTTPBasicAuth(username, password)) + if not r.ok: + raise Exception + + request = r.json() + + + import atexit + atexit.register(on_exit, username, password, request) + + ListenerWidget = QtWidgets.QWidget() + ui = Listener() + ui.setupUI(ListenerWidget, request['classes'], request['user']['first_name']) + ListenerWidget.show() + + sys.exit(app.exec_()) + + +class Listener(object): + def setupUI(self, Form, classes, name): + Form.setObjectName("Form") + Form.resize(340, 202) + self.name = name + self.horizontalLayoutWidget = QtWidgets.QWidget(Form) + self.horizontalLayoutWidget.setGeometry(QtCore.QRect(30, 10, 271, 121)) + self.horizontalLayoutWidget.setObjectName("horizontalLayoutWidget") + self.horizontalLayout = QtWidgets.QHBoxLayout(self.horizontalLayoutWidget) + self.horizontalLayout.setContentsMargins(0, 0, 0, 0) + self.horizontalLayout.setObjectName("horizontalLayout") + self.comboBox = QtWidgets.QComboBox(self.horizontalLayoutWidget) + self.comboBox.setObjectName("comboBox") + self.horizontalLayout.addWidget(self.comboBox) + self.play_pause = QtWidgets.QPushButton(Form) + self.play_pause.setGeometry(QtCore.QRect(50, 150, 241, 31)) + self.play_pause.setObjectName("play_pause") + self.play_pause.clicked.connect(self.start_listener) + + rooms = [f'Period {room["period"]} - {room["name"]} with {room["teacher"]} ID:({room["id"]})' for room in classes] + self.comboBox.addItems(rooms) + + self.retranslateUI(Form) + QtCore.QMetaObject.connectSlotsByName(Form) + + def retranslateUI(self, Form): + _translate = QtCore.QCoreApplication.translate + Form.setWindowTitle(_translate("SpaceOut", "SpaceOut")) + self.play_pause.setText(_translate("Form", "Start Listening")) + + + def start_listener(self): + choice = self.comboBox.currentText() + print(choice) + result = re.search('ID:(.*)', choice) + class_id = result.group(1)[1] + transcript = open('current_transcript', 'w') + transcript.write(f'{choice}-{str(datetime.datetime.now())}\n') + transcript.close() + text_to_speech.text_to_speech(self.name) + class LoginForm(object): - def setupUi(self, Form): + def setupUI(self, Form): Form.setObjectName("SpaceOut Login") Form.resize(500, 756) self.verticalLayout = QtWidgets.QVBoxLayout(Form) @@ -143,7 +239,7 @@ class LoginForm(object): self.horizontalLayout_3.addWidget(self.widget) self.verticalLayout.addLayout(self.horizontalLayout_3) - self.retranslateUi(Form) + self.retranslateUI(Form) QtCore.QMetaObject.connectSlotsByName(Form) def open_register_link(self): @@ -159,15 +255,15 @@ class LoginForm(object): f = open('.userinfo', 'w') f.write(self.username.text()) f.close() - app.quit() + main() else: msg = QtWidgets.QMessageBox() msg.setText('Incorrect Login') msg.exec_() - def retranslateUi(self, Form): + def retranslateUI(self, Form): _translate = QtCore.QCoreApplication.translate - Form.setWindowTitle(_translate("Form", "Form")) + Form.setWindowTitle(_translate("SpaceOut", "SpaceOut")) self.label.setText(_translate("Form", "

")) self.label_2.setText(_translate("Form", "

")) self.label_3.setText(_translate("Form", "

")) @@ -176,23 +272,7 @@ class LoginForm(object): import spaceout_rc + + if __name__ == "__main__": - import sys - app = QtWidgets.QApplication(sys.argv) - try: - f = open('.userinfo') - username = f.read() - f.close() - password = keyring.get_password('spaceout', username) - r = requests.get(f'{API_ENDPOINT}profile', auth=HTTPBasicAuth(username, password)) - if not r.ok: - raise Exception - - except: - Form = QtWidgets.QWidget() - ui = LoginForm() - ui.setupUi(Form) - Form.show() - - - sys.exit(app.exec_()) + main() diff --git a/app/text_to_speech.py b/app/text_to_speech.py new file mode 100644 index 00000000..dd62acec --- /dev/null +++ b/app/text_to_speech.py @@ -0,0 +1,252 @@ +import re +import os +import sys +import time +import winsound +import webbrowser +import threading +from threading import Thread + +from pywinauto import application +from pywinauto.findwindows import WindowAmbiguousError, WindowNotFoundError + +from google.cloud import speech +import pyaudio +from six.moves import queue + +# Audio recording parameters +STREAMING_LIMIT = 240000 +SAMPLE_RATE = 16000 +CHUNK_SIZE = int(SAMPLE_RATE / 10) + + +def get_current_time(): + return int(round(time.time() * 1000)) + + +class ResumableMicrophoneStream: + + def __init__(self, rate, chunk_size): + self._rate = rate + self.chunk_size = chunk_size + self._num_channels = 1 + self._buff = queue.Queue() + self.closed = True + self.start_time = get_current_time() + self.restart_counter = 0 + self.audio_input = [] + self.last_audio_input = [] + self.result_end_time = 0 + self.is_final_end_time = 0 + self.final_request_end_time = 0 + self.bridging_offset = 0 + self.last_transcript_was_final = False + self.new_stream = True + self._audio_interface = pyaudio.PyAudio() + self._audio_stream = self._audio_interface.open( + format=pyaudio.paInt16, + channels=self._num_channels, + rate=self._rate, + input=True, + frames_per_buffer=self.chunk_size, + stream_callback=self._fill_buffer, + ) + + def __enter__(self): + + self.closed = False + return self + + def __exit__(self, type, value, traceback): + + self._audio_stream.stop_stream() + self._audio_stream.close() + self.closed = True + self._buff.put(None) + self._audio_interface.terminate() + + def _fill_buffer(self, in_data, *args, **kwargs): + + self._buff.put(in_data) + return None, pyaudio.paContinue + + def generator(self): + + while not self.closed: + data = [] + + if self.new_stream and self.last_audio_input: + + chunk_time = STREAMING_LIMIT / len(self.last_audio_input) + + if chunk_time != 0: + + if self.bridging_offset < 0: + self.bridging_offset = 0 + + if self.bridging_offset > self.final_request_end_time: + self.bridging_offset = self.final_request_end_time + + chunks_from_ms = round( + (self.final_request_end_time - self.bridging_offset) + / chunk_time + ) + + self.bridging_offset = round( + (len(self.last_audio_input) - chunks_from_ms) * chunk_time + ) + + for i in range(chunks_from_ms, len(self.last_audio_input)): + data.append(self.last_audio_input[i]) + + self.new_stream = False + + chunk = self._buff.get() + self.audio_input.append(chunk) + + if chunk is None: + return + data.append(chunk) + # Now consume whatever other data's still buffered. + while True: + try: + chunk = self._buff.get(block=False) + + if chunk is None: + return + data.append(chunk) + self.audio_input.append(chunk) + + except queue.Empty: + break + + yield b"".join(data) + + +def listen_print_loop(responses, stream, name): + for response in responses: + + if get_current_time() - stream.start_time > STREAMING_LIMIT: + stream.start_time = get_current_time() + break + + if not response.results: + continue + + result = response.results[0] + + if not result.alternatives: + continue + + transcript = result.alternatives[0].transcript + + if result.is_final: + sys.stdout.write("Final: " + transcript + "\n") + + stream.is_final_end_time = stream.result_end_time + stream.last_transcript_was_final = True + transcript_file = open("current_transcript", "a") + transcript_file.write(f'{time.strftime("%H:%M:%S")} {transcript}\n') + transcript_file.close() + + if re.search(r"\b("+ name + r")\b", transcript, re.IGNORECASE): + name_called() + stream.closed = True + break + + else: + sys.stdout.write("Speaking: " + transcript + "\r") + + stream.last_transcript_was_final = False + + +def name_called(): + for i in range(2): + winsound.Beep(1500, 250) + winsound.Beep(600, 250) + + app = application.Application() + try: + app.connect(title_re=".*Chrome.*") + + app_dialog = app.top_window() + app_dialog.restore() + app_dialog.maximize() + + except(WindowNotFoundError): + print ("Couldn't open chrome") + pass + except(WindowAmbiguousError): + print ('There are too many Chrome windows found') + pass + + transcript = open('current_transcript', 'r') + last_spoken = '' + + for line in (transcript.readlines()[-3:]): + last_spoken += line + + print(last_spoken) + message_box(last_spoken) + + + +def message_box (text): + from PyQt5 import QtWidgets + msg = QtWidgets.QMessageBox() + msg.setText(text) + msg.show() + sys.exit(msg.exec_()) + +def text_to_speech(name): + + + """start bidirectional streaming from microphone input to speech API""" + os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = r"SpaceOut.json" + + + client = speech.SpeechClient() + config = speech.RecognitionConfig( + encoding=speech.RecognitionConfig.AudioEncoding.LINEAR16, + sample_rate_hertz=SAMPLE_RATE, + language_code="en-US", + max_alternatives=1, + ) + + streaming_config = speech.StreamingRecognitionConfig( + config=config, interim_results=True + ) + + mic_manager = ResumableMicrophoneStream(SAMPLE_RATE, CHUNK_SIZE) + + with mic_manager as stream: + + while not stream.closed: + + stream.audio_input = [] + audio_generator = stream.generator() + + requests = ( + speech.StreamingRecognizeRequest(audio_content=content) + for content in audio_generator + ) + + responses = client.streaming_recognize(streaming_config, requests) + + # Now, put the transcription responses to use. + listen_print_loop(responses, stream, name) + + if stream.result_end_time > 0: + stream.final_request_end_time = stream.is_final_end_time + stream.result_end_time = 0 + stream.last_audio_input = [] + stream.last_audio_input = stream.audio_input + stream.audio_input = [] + stream.restart_counter = stream.restart_counter + 1 + + stream.new_stream = True + + listen_print_loop(responses, stream, name) + +if __name__ == "__main__": + text_to_speech("lucas") # just for testing if script is directly invoked