From 5f98aa0ed844a71828e1f8d590b63d136e812167 Mon Sep 17 00:00:00 2001 From: Falko Habel Date: Thu, 14 Mar 2024 22:17:46 +0100 Subject: [PATCH] added HTTP Stream for response --- main.go | 245 ++++++++++++++++++++++++++++++++ translator/download.py | 34 +++++ translator/note_all_packages.py | 23 +++ translator/translate.py | 14 ++ 4 files changed, 316 insertions(+) create mode 100644 main.go create mode 100644 translator/download.py create mode 100644 translator/note_all_packages.py create mode 100644 translator/translate.py diff --git a/main.go b/main.go new file mode 100644 index 0000000..b9a14e3 --- /dev/null +++ b/main.go @@ -0,0 +1,245 @@ +package main + +import ( + "bufio" + "bytes" + "encoding/csv" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "os" + "os/exec" + "time" +) + +type Message struct { + Message string `json:"message"` + From string `json:"from"` + To string `json:"to"` +} + +// Note: The StreamResponse structure simplified for demonstration. Adjust according to actual requirements. +type StreamResponse struct { + From string `json:"from"` + To string `json:"to"` + CreatedAt string `json:"createdAt"` + Response string `json:"response"` +} + +func createStreamResponse(fromLanguage, toLanguage, message string) string { + response := StreamResponse{ + From: fromLanguage, + To: toLanguage, + CreatedAt: time.Now().Format(time.RFC3339), + Response: message, + } + jsonResp, err := json.Marshal(response) + if err != nil { + return `{"response": "Error in preparing the message."}` + } + return string(jsonResp) +} + +func streamResponse(w http.ResponseWriter, fromLanguage, toLanguage string, messages <-chan string) { + // Set headers for SSE + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + + flusher, ok := w.(http.Flusher) + if !ok { + http.Error(w, "Streaming unsupported!", http.StatusInternalServerError) + return + } + + for msg := range messages { + formattedMessage := createStreamResponse(fromLanguage, toLanguage, msg) + fmt.Fprintf(w, "data: %s\n\n", formattedMessage) + flusher.Flush() // Ensure client receives the update immediately + } +} + +func downloadPackages(w http.ResponseWriter, fromLanguage, toLanguage string) { + // Create a channel to send messages from the download process + messages := make(chan string) + defer close(messages) + + // Use a goroutine for streaming responses so we can proceed with the download + go streamResponse(w, fromLanguage, toLanguage, messages) + + // Start the Python command in the background + cmd := exec.Command("python", "translator/download.py", fromLanguage, toLanguage) + output, err := cmd.StdoutPipe() + if err != nil { + messages <- fmt.Sprintf("Error starting Download: %s", err.Error()) + return + } + + err = cmd.Start() + if err != nil { + messages <- fmt.Sprintf("Error starting Download: %s", err.Error()) + return + } + + // Initial message sent to the channel for streaming + messages <- "Download started . . ." + + // Read the output of the Python command and send it to the channel + scanner := bufio.NewScanner(output) + for scanner.Scan() { + messages <- scanner.Text() // Sends output line by line to the stream + } + + // Wait for the command to complete + err = cmd.Wait() + if err != nil { + messages <- fmt.Sprintf("Error waiting for Download: %s", err.Error()) + return + } +} + +func executeTranslator(w http.ResponseWriter, message, fromLanguage, toLanguage string) { + // Create a message channel for streaming translation results + messages := make(chan string) + defer close(messages) + + // A goroutine will manage sending streamed responses + go streamResponse(w, fromLanguage, toLanguage, messages) + + // Start the Python command in the background + cmd := exec.Command("python", "translator/translate.py", message, fromLanguage, toLanguage) + output, err := cmd.StdoutPipe() + if err != nil { + messages <- fmt.Sprintf("Error starting Translation: %s", err.Error()) + return + } + + err = cmd.Start() + if err != nil { + messages <- fmt.Sprintf("Error starting Translation: %s", err.Error()) + return + } + + // Read the output of the Python command and send it through the channel + scanner := bufio.NewScanner(output) + for scanner.Scan() { + messages <- scanner.Text() // Sends each line of the output to the stream + } + + // Wait for the command to complete + err = cmd.Wait() + if err != nil { + messages <- fmt.Sprintf("Error waiting for Translation: %s", err.Error()) + return + } +} + +func CheckLanguagesInstalled(fromCode, toCode string) (bool, error) { + // Open the CSV file + file, err := os.Open("data/installed_packages.csv") + if err != nil { + if os.IsNotExist(err) { + return false, fmt.Errorf("file not found") + } + return false, err + } + defer file.Close() + + // Create a new CSV reader + reader := csv.NewReader(file) + + // Iterate through the records + for { + record, err := reader.Read() + if err != nil { + if err == io.EOF { + break + } + return false, err + } + if len(record) >= 2 && record[0] == fromCode && record[1] == toCode { + return true, nil + } + } + + return false, nil +} +func getAllPackages() error { + cmd := exec.Command("python", "translator/note_all_packages.py") + // Create a buffer to capture the standard output. + var out bytes.Buffer + cmd.Stdout = &out + + // Execute the command. + err := cmd.Run() + if err != nil { + return err + } + + // Log the captured output. + fmt.Println("Output:", out.String()) + return nil +} +func handleRequest(w http.ResponseWriter, r *http.Request) { + // Read the request body + if r.Header.Get("Content-Type") != "application/json" { + http.Error(w, "Request content type must be application/json", http.StatusBadRequest) + return + } + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "Error reading request body: "+err.Error(), http.StatusInternalServerError) + return + } + defer r.Body.Close() + + // Unmarshal the JSON data + var msg Message + err = json.Unmarshal(body, &msg) + if err != nil { + http.Error(w, "Error unmarshalling JSON: "+err.Error(), http.StatusBadRequest) + return + } + + // Check if From and To fields are not longer than 2 letters + if len(msg.From) > 2 || len(msg.To) > 2 { + http.Error(w, "From and To fields should not be longer than 2 letters.", http.StatusBadRequest) + return + } else { + installed, err := CheckLanguagesInstalled(msg.From, msg.To) + if err != nil { + if err.Error() == "file not found" { + http.Error(w, "file not found "+err.Error(), http.StatusInternalServerError) + } else { + fmt.Println("An error occurred:", err) + } + return + } + if installed { + executeTranslator(w, msg.Message, msg.From, msg.To) + } else { + downloadPackages(w, msg.From, msg.To) + } + } +} + +func main() { + getAllPackages() + // Define the HTTP handler function + http.HandleFunc("/api", handleRequest) + + // Get the port number from the environment variable or use a default value + port := "11435" + if p := os.Getenv("PORT"); p != "" { + port = p + } + + // Start the HTTP server + fmt.Printf("Listening on :%s...\n", port) + err := http.ListenAndServe(":"+port, nil) + if err != nil { + log.Fatalf("Failed to start server: %v", err) + } +} diff --git a/translator/download.py b/translator/download.py new file mode 100644 index 0000000..d48c1d9 --- /dev/null +++ b/translator/download.py @@ -0,0 +1,34 @@ +import argostranslate.translate +import argostranslate.package +from note_all_packages import write_installed_packages_to_csv +import sys + +def install_language(): + from_lang = sys.argv[1] + to_lang = sys.argv[2] + try: + argostranslate.package.update_package_index() + available_packages = argostranslate.package.get_available_packages() + # Convert the filter result into a list + filtered_packages = list( + filter( + lambda x: x.from_code == from_lang and x.to_code == to_lang, available_packages + ) + ) + # Check if the filtered list is empty + if not filtered_packages: + return 1 + # If we have at least one package, proceed to download and install + + available_package = filtered_packages[0] + download_path = available_package.download() + argostranslate.package.install_from_path(download_path) + # After successful installation, append package details to the CSV file + write_installed_packages_to_csv() + print("Finished Download") + + except Exception as e: + return e + +if __name__ == "__main__": + install_language() diff --git a/translator/note_all_packages.py b/translator/note_all_packages.py new file mode 100644 index 0000000..6c5ee54 --- /dev/null +++ b/translator/note_all_packages.py @@ -0,0 +1,23 @@ +import argostranslate.package +import csv + + +# Function to write installed package details into a CSV file +def write_installed_packages_to_csv(filename='data/installed_packages.csv'): + # Fetch all installed packages + installed_packages = argostranslate.package.get_installed_packages() + # Open/Create a CSV file to write into + with open(filename, mode='w', newline='', encoding='utf-8') as file: + writer = csv.writer(file) + + # Write package details row by row + for package in installed_packages: + from_language = package.from_code + to_language = package.to_code + package_name = package.from_name + + writer.writerow([from_language, to_language, package_name]) + +if __name__ == "__main__": + write_installed_packages_to_csv() + \ No newline at end of file diff --git a/translator/translate.py b/translator/translate.py new file mode 100644 index 0000000..9ea63f2 --- /dev/null +++ b/translator/translate.py @@ -0,0 +1,14 @@ +import argostranslate.package +import argostranslate.translate +import sys + +def translate(): + message = sys.argv[1] + from_lang = sys.argv[2] + to_lang = sys.argv[3] + translation = argostranslate.translate.translate(message, from_lang, to_lang) + print(translation) + + +if __name__ == "__main__": + translate()