import streamlit as st import sparknlp from sparknlp.base import * from sparknlp.annotator import * from pyspark.ml import Pipeline # Page configuration st.set_page_config( layout="wide", initial_sidebar_state="auto" ) # CSS for styling st.markdown(""" """, unsafe_allow_html=True) @st.cache_resource def init_spark(): return sparknlp.start() @st.cache_resource def create_pipeline(model): document_assembler = DocumentAssembler() \ .setInputCol("text") \ .setOutputCol("documents") sentence_detector = SentenceDetectorDLModel\ .pretrained()\ .setInputCols(["documents"])\ .setOutputCol("questions") t5 = T5Transformer()\ .pretrained("google_t5_small_ssm_nq")\ .setInputCols(["questions"])\ .setOutputCol("answers")\ pipeline = Pipeline().setStages([document_assembler, sentence_detector, t5]) return pipeline def fit_data(pipeline, data): df = spark.createDataFrame([[data]]).toDF("text") result = pipeline.fit(df).transform(df) return result.select('answers.result').collect() # Sidebar content model = st.sidebar.selectbox( "Choose the pretrained model", ['google_t5_small_ssm_nq'], help="For more info about the models visit: https://sparknlp.org/models" ) # Set up the page layout title, sub_title = ( 'Automatically Answer Questions (CLOSED BOOK)', 'Automatically generate answers to questions without context.' ) st.markdown(f'