Update app.py

app.py

@@ -1,14 +1,24 @@
 import streamlit as st
+import polars as pl
 from transformers import T5Tokenizer, T5ForConditionalGeneration, AutoTokenizer, AutoModelForSeq2SeqLM, pipeline, logging, AutoModelForCausalLM
 import torch
 import os
 import httpx
 
-logging.set_verbosity_error()
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 # Language options and mappings
 options = ["German", "Romanian", "English", "French", "Spanish", "Italian"]
 langs = {"English": "en", "Romanian": "ro", "German": "de", "French": "fr", "Spanish": "es", "Italian": "it"}
+favourite_langs = {"German": "de", "Romanian": "ro", "English": "en", "-----": "-----"}
+df = pl.read_parquet("isolanguages.parquet")
+non_empty_isos = df.slice(1).filter(pl.col("ISO639-1") != "").rows()
+# all_langs = languagecodes.iso_languages_byname
+all_langs = {iso[0]: (iso[1], iso[2], iso[3]) for iso in non_empty_isos} # {'Romanian': ('ro', 'rum', 'ron')}
+# iso1_to_name = {codes[0]: lang for entry in all_langs for lang, codes in entry.items()} # {'ro': 'Romanian', 'de': 'German'}
+iso1_to_name = {iso[1]: iso[0] for iso in non_empty_isos} # {'ro': 'Romanian', 'de': 'German'}
+langs = list(favourite_langs.keys())
+langs.extend(list(all_langs.keys())) # Language options as list, add favourite languages first
+
 models = ["Helsinki-NLP", "Argos", "t5-base", "t5-small", "t5-large", "Unbabel/Tower-Plus-2B", 
           "Unbabel/TowerInstruct-Mistral-7B-v0.2", "winninghealth/WiNGPT-Babel-2", "Google"]
 allmodels = ["Helsinki-NLP",