pages/0_KOSMOS-2.py

import streamlit as st
from streamlit_extras.switch_page_button import switch_page


translations = {
'en': {'title': 'KOSMOS-2',
    'original_tweet': 
       """
       [Original tweet](https://x.com/mervenoyann/status/1720126908384366649) (November 2, 2023)
       """,
    'tweet_1':
        """
        New 🤗 Transformers release includes a very powerful Multimodel Large Language Model (MLLM) by @Microsoft called KOSMOS-2! 🤩  
        The highlight of KOSMOS-2 is grounding, the model is *incredibly* accurate! 🌎  
        Play with the demo [here](https://huggingface.co/spaces/ydshieh/Kosmos-2) by [@ydshieh](https://x.com/ydshieh).  
        But how does this model work? Let's take a look! 👀🧶
        """,
    'tweet_2':
        """
        Grounding helps machine learning models relate to real-world examples. Including grounding makes models more performant by means of accuracy and robustness during inference. It also helps reduce the so-called "hallucinations" in language models.
        """,
    'tweet_3':
        """
        In KOSMOS-2, model is grounded to perform following tasks and is evaluated on 👇  
        - multimodal grounding  &  phrase grounding, e.g. localizing the object through natural language query  
        - multimodal referring, e.g. describing object characteristics & location  
        - perception-language tasks  
        - language understanding and generation
        """,
    'tweet_4':
        """
        The dataset used for grounding, called GRiT is also available on [Hugging Face Hub](https://huggingface.co/datasets/zzliang/GRIT).  
        Thanks to 🤗 Transformers integration, you can use KOSMOS-2 with few lines of code 🤩  
        See below! 👇
        """,
    'ressources':
        """
        Ressources:  
        [Kosmos-2: Grounding Multimodal Large Language Models to the World](https://arxiv.org/abs/2306.14824) 
        by Zhiliang Peng, Wenhui Wang, Li Dong, Yaru Hao, Shaohan Huang, Shuming Ma, Furu Wei (2023)  
        [GitHub](https://github.com/microsoft/unilm/tree/master/kosmos-2)  
	[Hugging Face documentation](https://huggingface.co/docs/transformers/model_doc/kosmos-2)
        """
      },
'fr': {
    'title': 'KOSMOS-2',
    'original_tweet': 
       """
       [Tweet de base](https://x.com/mervenoyann/status/1720126908384366649) (en anglais) (2 novembre 2023)
       """,
    'tweet_1':
        """
        La nouvelle version de 🤗 Transformers inclut un très puissant <i>Multimodel Large Language Model</i> (MLLM) de @Microsoft appelé KOSMOS-2 ! 🤩  
        Le point fort de KOSMOS-2 est l'ancrage, le modèle est *incroyablement* précis ! 🌎  
        Jouez avec la démo [ici](https://huggingface.co/spaces/ydshieh/Kosmos-2) de [@ydshieh](https://x.com/ydshieh).  
        Mais comment fonctionne t'il ? Jetons un coup d'œil ! 👀🧶
        """,
    'tweet_2':
        """
        L'ancrage permet aux modèles d'apprentissage automatique d'être liés à des exemples du monde réel. L'inclusion de l'ancrage rend les modèles plus performants en termes de précision et de robustesse lors de l'inférence. Cela permet également de réduire les « hallucinations » dans les modèles de langage.        """,
    'tweet_3':
        """
        Dans KOSMOS-2, le modèle est ancré pour effectuer les tâches suivantes et est évalué sur 👇  
        - l'ancrage multimodal et l'ancrage de phrases, par exemple la localisation de l'objet par le biais d'une requête en langage naturel  
        - la référence multimodale, par exemple la description des caractéristiques et de l'emplacement de l'objet  
        - tâches de perception-langage  
        - compréhension et génération du langage
        """,
    'tweet_4':
        """
        Le jeu de données utilisé pour l'ancrage, appelé GRiT, est également disponible sur le [Hub d'Hugging Face](https://huggingface.co/datasets/zzliang/GRIT).  
        Grâce à l'intégration dans 🤗 Transformers, vous pouvez utiliser KOSMOS-2 avec quelques lignes de code 🤩.  
        Voir ci-dessous ! 👇
        """,
    'ressources':
        """
        Ressources :  
        [Kosmos-2: Grounding Multimodal Large Language Models to the World](https://arxiv.org/abs/2306.14824) 
        de Zhiliang Peng, Wenhui Wang, Li Dong, Yaru Hao, Shaohan Huang, Shuming Ma, Furu Wei (2023)  
        [GitHub](https://github.com/microsoft/unilm/tree/master/kosmos-2)  
	[Documentation d'Hugging Face](https://huggingface.co/docs/transformers/model_doc/kosmos-2)
        """
    }
}    


def language_selector():
    languages = {'EN': '🇬🇧', 'FR': '🇫🇷'}
    selected_lang = st.selectbox('', options=list(languages.keys()), format_func=lambda x: languages[x], key='lang_selector')
    return 'en' if selected_lang == 'EN' else 'fr'

left_column, right_column = st.columns([5, 1])

# Add a selector to the right column
with right_column:
    lang = language_selector()

# Add a title to the left column
with left_column:
    st.title(translations[lang]["title"])
    
st.success(translations[lang]["original_tweet"], icon="ℹ️")
st.markdown(""" """)

st.markdown(translations[lang]["tweet_1"], unsafe_allow_html=True)
st.markdown(""" """)

st.video("pages/KOSMOS-2/video_1.mp4", format="video/mp4")
st.markdown(""" """)

st.markdown(translations[lang]["tweet_2"], unsafe_allow_html=True)
st.markdown(""" """)

st.markdown(translations[lang]["tweet_3"], unsafe_allow_html=True)
st.markdown(""" """)

st.markdown(translations[lang]["tweet_4"], unsafe_allow_html=True)
st.markdown(""" """)

st.image("pages/KOSMOS-2/image_1.jpg", use_container_width=True)
st.markdown(""" """)

with st.expander ("Code"):
    if lang == "en":
        st.code("""
        from transformers import AutoProcessor, AutoModelForVision2Seq 
    
        model = AutoModelForVision2Seq.from_pretrained("microsoft/kosmos-2-patch14-224").to("cuda") 
        processor = AutoProcessor.from_pretrained("microsoft/kosmos-2-patch14-224") 
    
        image_input = Image.open(user_image_path) 
        # prepend different preprompts optionally to describe images
        brief_preprompt = "<grounding>An image of" 
        detailed_preprompt = "<grounding>Describe this image in detail:" 
    
      
        inputs = processor(text=text_input, images=image_input, return_tensors="pt").to("cuda") 
    
        generated_ids = model.generate( 
            pixel_values=inputs["pixel_values"],
            input_ids=inputs["input_ids"], 
            attention_mask=inputs["attention_mask"], 
            image_embeds=None,
            image_embeds_position_mask=inputs["image_embeds_position_mask"], 
            use_cache=True,
            max_new_tokens=128,
        ) 
    
        generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] 
        
        processed_text, entities = processor.post_process_generation(generated_text)
        
        # check out the Space for inference with bbox drawing
        """)
    else:
        st.code("""
        from transformers import AutoProcessor, AutoModelForVision2Seq 
    
        model = AutoModelForVision2Seq.from_pretrained("microsoft/kosmos-2-patch14-224").to("cuda") 
        processor = AutoProcessor.from_pretrained("microsoft/kosmos-2-patch14-224") 
    
        image_input = Image.open(user_image_path) 
        # ajouter différents préprompts facultatifs pour décrire les images
        brief_preprompt = "<grounding>An image of" 
        detailed_preprompt = "<grounding>Describe this image in detail:" 
    
      
        inputs = processor(text=text_input, images=image_input, return_tensors="pt").to("cuda") 
    
        generated_ids = model.generate( 
            pixel_values=inputs["pixel_values"],
            input_ids=inputs["input_ids"], 
            attention_mask=inputs["attention_mask"], 
            image_embeds=None,
            image_embeds_position_mask=inputs["image_embeds_position_mask"], 
            use_cache=True,
            max_new_tokens=128,
        ) 
    
        generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] 
        
        processed_text, entities = processor.post_process_generation(generated_text)
        
        # consultez le Space pour l'inférence avec le tracé des bbox
        """)
st.markdown(""" """)

st.info(translations[lang]["ressources"], icon="📚")  

st.markdown(""" """)
st.markdown(""" """)
st.markdown(""" """)
col1, col2, col3= st.columns(3)
with col1:
    if lang == "en":
        if st.button('Previous paper', use_container_width=True):
            switch_page("Home")
    else:
        if st.button('Papier précédent', use_container_width=True):
            switch_page("Home")
with col2:
    if lang == "en":
        if st.button("Home", use_container_width=True):
            switch_page("Home")
    else:
        if st.button("Accueil", use_container_width=True):
            switch_page("Home")
with col3:
    if lang == "en":
        if st.button("Next paper", use_container_width=True):
            switch_page("MobileSAM")
    else:
        if st.button("Papier suivant", use_container_width=True):
            switch_page("MobileSAM")