Futurebot/Labtourmode.py

import pygame
import speech_recognition as sr
from openai import OpenAI
from pathlib import Path
import time
import io
import soundfile as sf
import sounddevice as sd
import random
import csv
import cv2
from pyzbar.pyzbar import decode
from picamera2 import Picamera2

# Initialize OpenAI API key
api_key = 'sk-proj-wwWaxim1Qt13uqzSS054xjT3BlbkFJK0r78z676Zvx78AJiWG3Ot7d3S'
client = OpenAI(api_key=api_key)

# Global variable to store CSV content
csv_content = ""

def create_messages(question, file_content):
    return [
        {"role": "system", "content": "Your name is Futurebot. You were created by Sooraj and team who develops innovative projects in IoT future lab at Vodafone. You work with Tim, Sooraj, and Priya along with other team members Laura, Sven, Thomas, and Stephie. You are from T-E-T-I team. Your manager is Teja. You are a lab tour guide who explains and answers about IoT use cases in Vodafone. You have to create and complete explanations and answers in a meaningful way under 150 tokens. Do not say greetings. Do not say any calculations. Directly say the result battery level in percentage without decimal values. You should only answer about the loaded text and its associated data."},
        {"role": "user", "content": f"{file_content}\n\nQ: {question}\nA:"}
    ]

def play_audio(file_name):
    if file_name:
        print(f"Playing audio: {file_name}")  # Debugging print
        pygame.mixer.quit()  # Reset the mixer
        pygame.mixer.init()  # Reinitialize
        pygame.mixer.music.load(file_name)
        pygame.mixer.music.play()
        while pygame.mixer.music.get_busy():
            time.sleep(1)  # Ensures the program waits until audio finishes


def recognize_speech():
    recognizer = sr.Recognizer()
    with sr.Microphone() as source:
        print("Listening...")
        audio = recognizer.listen(source)
        try:
            print("Recognizing...")
            text = recognizer.recognize_google(audio, language='en-US')
            audio_files = ["ty.mp3", "th.mp3", "good.mp3", "hmm.mp3"]
            random_audio = random.choice(audio_files)
            pygame.mixer.music.load(random_audio)
            pygame.mixer.music.play()
            print(f"You said: {text}")
            return text
        except sr.UnknownValueError:
            print("Sorry, I did not understand that.")
            return None
        except sr.RequestError:
            print("Sorry, there was an error with the speech recognition service.")
            return None

def get_response_from_openai(messages):
    stream = client.chat.completions.create(
        model="gpt-3.5-turbo",
        max_tokens=150,
        temperature=0.5,
        messages=messages,
        stream=True,
    )
    for chunk in stream:
        if chunk.choices[0].delta.content is not None:
            yield chunk.choices[0].delta.content

def read_file(file_path):
    if file_path.endswith('.txt'):
        with open(file_path, 'r') as file:
            return file.read()
    elif file_path.endswith('.csv'):
        content = []
        with open(file_path, 'r') as file:
            reader = csv.reader(file)
            for row in reader:
                content.append(', '.join(row))
        return '\n'.join(content)
    else:
        raise ValueError("Unsupported file format")

def generate_speech(text):
    if text.strip():
        spoken_response = client.audio.speech.create(
            model="tts-1",
            voice="alloy",
            input=text
        )

        buffer = io.BytesIO()
        for chunk in spoken_response.iter_bytes(chunk_size=4096):
            buffer.write(chunk)
        buffer.seek(0)

        with sf.SoundFile(buffer, 'r') as sound_file:
            data = sound_file.read(dtype='int16')
            sd.play(data, sound_file.samplerate)
            sd.wait()

def scan_qr_code(image):
    """Scans the QR code from the given image and returns the decoded data."""
    qr_codes = decode(image)
    if qr_codes:
        qr_data = qr_codes[0].data.decode('utf-8')
        print("QR Code detected:", qr_data)
        return qr_data
    return None

def start_qa_mode(file_content):
    while True:
        question = recognize_speech()

        if question and question.lower() in ["no", "go to the next showcase", "exit", "i don't have any questions", "i have no questions", "i don't have any other questions", "I don't have any other questions", "thats it"]:
            pygame.mixer.music.load("give.mp3")
            pygame.mixer.music.play()
            break

        if question:
            messages = create_messages(question, file_content)
            response_generator = get_response_from_openai(messages)
            print("Answer: ", end="")
            accumulated_response = ""
            for response_chunk in response_generator:
                accumulated_response += response_chunk
                if '.' in response_chunk or len(accumulated_response) > 300:
                    print(accumulated_response, end="", flush=True)
                    generate_speech(accumulated_response)
                    accumulated_response = ""
            if accumulated_response:
                print(accumulated_response, end="", flush=True)
                generate_speech(accumulated_response)
        else:
            print("Sorry, I didn't get that. Please ask again.")

def load_csv_content():
    global csv_content
    csv_file_path = "device_data.csv"  # Set your CSV file path here
    csv_content = read_file(csv_file_path)

def main():
    picam2 = Picamera2()
    picam2.start()

    text_files = {
        "gigabeeprotect": "Gigabeeprotect.txt",
        "hydrosense": "Hydrosense.txt",
        "pushtotalk": "Pushtotalk.txt",
        "rfid": "RFIDautomationenabler.txt",
        "smartsanitiser": "smartsanitiserdispenser.txt",
        "iotconfigurator": "IoTConfiguratorSolution.txt",
        "networkanalyser": "Networkanalyser.txt",
    }

    audio_files = {
        "gigabeeprotect": "Gigabee.mp3",
        "hydrosense": "speech1.mp3",
        "pushtotalk": "Pushtotalk.mp3",
        "rfid": "RFID.mp3",
        "smartsanitiser": "smartsanitiser.mp3",
        "iotconfigurator": "iotconfig.mp3",
        "networkanalyser": "NetworkAnalyser.mp3",
    }

    load_csv_content()

    while True:
        # Capture image from camera
        image = picam2.capture_array()

        # Scan QR code
        qr_data = scan_qr_code(image)

        if qr_data and qr_data in text_files:
            # Play associated audio and load the text file content
            play_audio(audio_files[qr_data])
            file_content = read_file(text_files[qr_data])
            combined_content = file_content + '\n\n' + csv_content  # Combine text file content with CSV content
            start_qa_mode(combined_content)

        cv2.imshow('QR Code Scanner', image)

        if cv2.waitKey(1) & 0xFF == ord('q'):
            break

    picam2.stop()
    cv2.destroyAllWindows()
    pygame.quit()

if __name__ == "__main__":
    main()