Update to upstream

pages/1 Scattertext.py

@@ -361,8 +361,9 @@ if uploaded_file is not None:
                     'Choose column',
                     (df_col_sel), on_change=reset_all)
         
-                list_words = paper[column_selected].values.tolist()
-                list_unique = sorted(list(set(list_words)))
+                list_words = paper[column_selected].dropna()  # remove NaN
+                list_words = [w for w in list_words if str(w).strip() != ""]  # remove empty strings
+                list_unique = sorted(set(list_words))
                 
                 if column_selected is not None:
                     label1 = col2.selectbox(
@@ -444,7 +445,7 @@ if uploaded_file is not None:
             st.write("Click the :blue[Download SVG] on the right side.")  
             st.divider()
             st.subheader(':blue[Scattertext Dataframe]', anchor=False)
-            st.button('📥 Click to download result')
+            st.button('📥 Click to download result', on_click="ignore")
             st.text("Click the Download button to get the CSV result.")
 
     except NameError:

pages/2 Topic Modeling.py

@@ -1,671 +1,674 @@
-#import module
-import streamlit as st
-import streamlit.components.v1 as components
-import pandas as pd
-import numpy as np
-import re
-import string
-import nltk
-nltk.download('wordnet')
-from nltk.stem import WordNetLemmatizer
-nltk.download('stopwords')
-from nltk.corpus import stopwords
-import gensim
-import gensim.corpora as corpora
-from gensim.corpora import Dictionary
-from gensim.models.coherencemodel import CoherenceModel
-from gensim.models.ldamodel import LdaModel
-from gensim.models import Phrases
-from gensim.models.phrases import Phraser
-from pprint import pprint
-import pickle
-import pyLDAvis
-import pyLDAvis.gensim_models as gensimvis
-from io import StringIO
-from ipywidgets.embed import embed_minimal_html
-from nltk.stem.snowball import SnowballStemmer
-from bertopic import BERTopic
-from bertopic.representation import KeyBERTInspired, MaximalMarginalRelevance, OpenAI, TextGeneration
-import plotly.express as px
-from sklearn.cluster import KMeans
-from sklearn.feature_extraction.text import CountVectorizer
-import bitermplus as btm
-import tmplot as tmp
-import tomotopy
-import sys
-import spacy
-import en_core_web_sm
-import pipeline
-from html2image import Html2Image
-from umap import UMAP
-import os
-import time
-import json
-from tools import sourceformat as sf
-import datamapplot
-from sentence_transformers import SentenceTransformer
-import openai
-from transformers import pipeline
-
-#===config===
-st.set_page_config(
-    page_title="Coconut",
-    page_icon="🥥",
-    layout="wide",
-    initial_sidebar_state="collapsed"
-)
-
-hide_streamlit_style = """
-            <style>
-            #MainMenu 
-            {visibility: hidden;}
-            footer {visibility: hidden;}
-            [data-testid="collapsedControl"] {display: none}
-            </style>
-            """
-st.markdown(hide_streamlit_style, unsafe_allow_html=True)
-
-with st.popover("🔗 Menu"):
-    st.page_link("https://www.coconut-libtool.com/", label="Home", icon="🏠")
-    st.page_link("pages/1 Scattertext.py", label="Scattertext", icon="1️⃣")
-    st.page_link("pages/2 Topic Modeling.py", label="Topic Modeling", icon="2️⃣")
-    st.page_link("pages/3 Bidirected Network.py", label="Bidirected Network", icon="3️⃣")
-    st.page_link("pages/4 Sunburst.py", label="Sunburst", icon="4️⃣")
-    st.page_link("pages/5 Burst Detection.py", label="Burst Detection", icon="5️⃣")
-    st.page_link("pages/6 Keywords Stem.py", label="Keywords Stem", icon="6️⃣")
-    st.page_link("pages/7 Sentiment Analysis.py", label="Sentiment Analysis", icon="7️⃣")
-    st.page_link("pages/8 Shifterator.py", label="Shifterator", icon="8️⃣")
-    st.page_link("pages/9 Summarization.py", label = "Summarization",icon ="9️⃣")
-    st.page_link("pages/10 WordCloud.py", label = "WordCloud", icon = "🔟")
-
-st.header("Topic Modeling", anchor=False)
-st.subheader('Put your file here...', anchor=False)
-
-#========unique id========
-@st.cache_resource(ttl=3600)
-def create_list():
-    l = [1, 2, 3]
-    return l
-
-l = create_list()
-first_list_value = l[0]
-l[0] = first_list_value + 1
-uID = str(l[0])
-
-@st.cache_data(ttl=3600)
-def get_ext(uploaded_file):
-    extype = uID+uploaded_file.name
-    return extype
-
-#===clear cache===
-
-def reset_biterm():
-    try:
-        biterm_map.clear()
-        biterm_bar.clear()
-    except NameError:
-        biterm_topic.clear()
-
-def reset_all():
-    st.cache_data.clear()
-
-#===avoiding deadlock===
-os.environ["TOKENIZERS_PARALLELISM"] = "false"
-        
-#===upload file===
-@st.cache_data(ttl=3600)
-def upload(file):
-    papers = pd.read_csv(uploaded_file)
-    if "About the data" in papers.columns[0]:
-        papers = sf.dim(papers)
-        col_dict = {'MeSH terms': 'Keywords',
-        'PubYear': 'Year',
-        'Times cited': 'Cited by',
-        'Publication Type': 'Document Type'
-        }
-        papers.rename(columns=col_dict, inplace=True)
-    
-    return papers
-
-@st.cache_data(ttl=3600)
-def conv_txt(extype):
-    if("PMID" in (uploaded_file.read()).decode()):
-        uploaded_file.seek(0)
-        papers = sf.medline(uploaded_file)
-        print(papers)
-        return papers
-    col_dict = {'TI': 'Title',
-            'SO': 'Source title',
-            'DE': 'Author Keywords',
-            'DT': 'Document Type',
-            'AB': 'Abstract',
-            'TC': 'Cited by',
-            'PY': 'Year',
-            'ID': 'Keywords Plus',
-            'rights_date_used': 'Year'}
-    uploaded_file.seek(0)
-    papers = pd.read_csv(uploaded_file, sep='\t')
-    if("htid" in papers.columns):
-        papers = sf.htrc(papers)
-    papers.rename(columns=col_dict, inplace=True)
-    print(papers)
-    return papers
-
-
-@st.cache_data(ttl=3600)
-def conv_json(extype):
-    col_dict={'title': 'title',
-    'rights_date_used': 'Year',
-    }
-
-    data = json.load(uploaded_file)
-    hathifile = data['gathers']
-    keywords = pd.DataFrame.from_records(hathifile)
-    
-    keywords = sf.htrc(keywords)
-    keywords.rename(columns=col_dict,inplace=True)
-    return keywords
-
-@st.cache_resource(ttl=3600)
-def conv_pub(extype):
-    if (get_ext(extype)).endswith('.tar.gz'):
-        bytedata = extype.read()
-        keywords = sf.readPub(bytedata)
-    elif (get_ext(extype)).endswith('.xml'):
-        bytedata = extype.read()
-        keywords = sf.readxml(bytedata)
-    return keywords
-
-#===Read data===
-uploaded_file = st.file_uploader('', type=['csv', 'txt','json','tar.gz','xml'], on_change=reset_all)
-
-if uploaded_file is not None:
-    try:
-        extype = get_ext(uploaded_file)
-    
-        if extype.endswith('.csv'):
-             papers = upload(extype) 
-        elif extype.endswith('.txt'):
-             papers = conv_txt(extype)
-
-        elif extype.endswith('.json'):
-            papers = conv_json(extype)
-        elif extype.endswith('.tar.gz') or extype.endswith('.xml'):
-            papers = conv_pub(uploaded_file)
-
-        coldf = sorted(papers.select_dtypes(include=['object']).columns.tolist())
-            
-        c1, c2, c3 = st.columns([3,3,4])
-        method = c1.selectbox(
-                'Choose method',
-                ('Choose...', 'pyLDA', 'Biterm', 'BERTopic'))
-        ColCho = c2.selectbox('Choose column', (["Abstract","Title", "Abstract + Title"]))
-        num_cho = c3.number_input('Choose number of topics', min_value=2, max_value=30, value=5)
-
-        d1, d2 = st.columns([3,7])
-        xgram = d1.selectbox("N-grams", ("1", "2", "3"))
-        xgram = int(xgram)
-        words_to_remove = d2.text_input("Remove specific words. Separate words by semicolons (;)")
-    
-        rem_copyright = d1.toggle('Remove copyright statement', value=True)
-        rem_punc = d2.toggle('Remove punctuation', value=True)
-
-        #===advance settings===
-        with st.expander("🧮 Show advance settings"): 
-            t1, t2, t3 = st.columns([3,3,4])
-            if method == 'pyLDA':
-                py_random_state = t1.number_input('Random state', min_value=0, max_value=None, step=1)
-                py_chunksize = t2.number_input('Chunk size', value=100 , min_value=10, max_value=None, step=1)
-                opt_threshold = t3.number_input('Threshold', value=100 , min_value=1, max_value=None, step=1)
-                
-            elif method == 'Biterm':
-                btm_seed = t1.number_input('Random state seed', value=100 , min_value=1, max_value=None, step=1)
-                btm_iterations = t2.number_input('Iterations number', value=20 , min_value=2, max_value=None, step=1)
-                opt_threshold = t3.number_input('Threshold', value=100 , min_value=1, max_value=None, step=1)
-                
-            elif method == 'BERTopic':
-                u1, u2 = st.columns([5,5])
-                
-                bert_top_n_words = u1.number_input('top_n_words', value=5 , min_value=5, max_value=25, step=1)
-                bert_random_state = u2.number_input('random_state', value=42 , min_value=1, max_value=None, step=1)
-                bert_n_components = u1.number_input('n_components', value=5 , min_value=1, max_value=None, step=1)
-                bert_n_neighbors = u2.number_input('n_neighbors', value=15 , min_value=1, max_value=None, step=1)
-                bert_embedding_model = st.radio(
-                    "embedding_model", 
-                    ["all-MiniLM-L6-v2", "paraphrase-multilingual-MiniLM-L12-v2", "en_core_web_sm"], index=0, horizontal=True)
-                
-                fine_tuning = st.toggle("Use Fine-tuning")
-                if fine_tuning:
-                    topic_labelling = st.toggle("Automatic topic labelling")
-                    if topic_labelling:
-                        llm_provider = st.selectbox("Model",["OpenAI/gpt-4o","Google/flan-t5","LiquidAI/LFM2-350M"])
-                        if llm_provider == "OpenAI/gpt-4o":
-                            api_key = st.text_input("API Key")
-                
-            else:
-                st.write('Please choose your preferred method')
-
-        #===clean csv===
-        @st.cache_data(ttl=3600, show_spinner=False)
-        def clean_csv(extype):
-            if (ColCho=="Abstract + Title"):
-                papers["Abstract + Title"] = papers["Title"] + " " + papers["Abstract"]
-                st.write(papers["Abstract + Title"])
-
-            paper = papers.dropna(subset=[ColCho])
-                     
-            #===mapping===
-            paper['Abstract_pre'] = paper[ColCho].map(lambda x: x.lower())
-            if rem_punc:
-                paper['Abstract_pre'] = paper['Abstract_pre'].map(
-                    lambda x: re.sub(f"[{re.escape(string.punctuation)}]", " ", x)
-                ).map(lambda x: re.sub(r"\s+", " ", x).strip())
-                paper['Abstract_pre'] = paper['Abstract_pre'].str.replace('[\u2018\u2019\u201c\u201d]', '', regex=True)
-            if rem_copyright:  
-                paper['Abstract_pre'] = paper['Abstract_pre'].map(lambda x: re.sub('©.*', '', x))
-            
-            #===stopword removal===
-            stop = stopwords.words('english')
-            paper['Abstract_stop'] = paper['Abstract_pre'].apply(lambda x: ' '.join([word for word in x.split() if word not in (stop)]))
-              
-            #===lemmatize===
-            lemmatizer = WordNetLemmatizer()
-            
-            @st.cache_resource(ttl=3600)
-            def lemmatize_words(text):
-                words = text.split()
-                words = [lemmatizer.lemmatize(word) for word in words]
-                return ' '.join(words)
-            paper['Abstract_lem'] = paper['Abstract_stop'].apply(lemmatize_words)
-        
-            words_rmv = [word.strip() for word in words_to_remove.split(";")]
-            remove_dict = {word: None for word in words_rmv}
-            
-            @st.cache_resource(ttl=3600)
-            def remove_words(text):
-                 words = text.split()
-                 cleaned_words = [word for word in words if word not in remove_dict]
-                 return ' '.join(cleaned_words) 
-            paper['Abstract_lem'] = paper['Abstract_lem'].map(remove_words)
-             
-            topic_abs = paper.Abstract_lem.values.tolist()
-            return topic_abs, paper
-    
-        topic_abs, paper=clean_csv(extype) 
-                 
-        if st.button("Submit", on_click=reset_all):
-            num_topic = num_cho  
-    
-        if method == 'BERTopic':
-            st.info('BERTopic is an expensive process when dealing with a large volume of text with our existing resources. Please kindly wait until the visualization appears.', icon="ℹ️")
-               
-        #===topic===
-        if method == 'Choose...':
-            st.write('')
-    
-        elif method == 'pyLDA':       
-            tab1, tab2, tab3, tab4 = st.tabs(["📈 Generate visualization", "📃 Reference", "📓 Recommended Reading", "⬇️ Download Help"])
-    
-            with tab1:
-            #===visualization===
-                @st.cache_data(ttl=3600, show_spinner=False)
-                def pylda(extype):
-                    topic_abs_LDA = [t.split(' ') for t in topic_abs]
-
-                    bigram = Phrases(topic_abs_LDA, min_count=xgram, threshold=opt_threshold)
-                    trigram = Phrases(bigram[topic_abs_LDA], threshold=opt_threshold)
-                    bigram_mod = Phraser(bigram)
-                    trigram_mod = Phraser(trigram)
-                    
-                    topic_abs_LDA = [trigram_mod[bigram_mod[doc]] for doc in topic_abs_LDA]
-
-                    id2word = Dictionary(topic_abs_LDA)
-                    corpus = [id2word.doc2bow(text) for text in topic_abs_LDA]
-                    #===LDA===
-                    lda_model = LdaModel(corpus=corpus,
-                                id2word=id2word,
-                                num_topics=num_topic, 
-                                random_state=py_random_state,
-                                chunksize=py_chunksize,
-                                alpha='auto',
-                                per_word_topics=False)
-                    pprint(lda_model.print_topics())
-                    doc_lda = lda_model[corpus]
-                    topics = lda_model.show_topics(num_words = 30,formatted=False)
-
-                    #===visualization===
-                    coherence_model_lda = CoherenceModel(model=lda_model, texts=topic_abs_LDA, dictionary=id2word, coherence='c_v')
-                    coherence_lda = coherence_model_lda.get_coherence()
-                    vis = pyLDAvis.gensim_models.prepare(lda_model, corpus, id2word)
-                    py_lda_vis_html = pyLDAvis.prepared_data_to_html(vis)
-                    return py_lda_vis_html, coherence_lda, vis, topics
-                       
-                with st.spinner('Performing computations. Please wait ...'):
-                    try:
-                        py_lda_vis_html, coherence_lda, vis, topics = pylda(extype)
-                        st.write('Coherence score: ', coherence_lda)
-                        components.html(py_lda_vis_html, width=1500, height=800)
-                        st.markdown('Copyright (c) 2015, Ben Mabey. https://github.com/bmabey/pyLDAvis')
-                           
-                        @st.cache_data(ttl=3600, show_spinner=False)
-                        def img_lda(vis):
-                            pyLDAvis.save_html(vis, 'output.html')
-                            hti = Html2Image()
-                            hti.browser.flags = ['--default-background-color=ffffff', '--hide-scrollbars']
-                            hti.browser.use_new_headless = None
-                            css = "body {background: white;}"
-                            hti.screenshot( 
-                                other_file='output.html', css_str=css, size=(1500, 800),
-                                save_as='ldavis_img.png'
-                            )
-
-                        img_lda(vis)
-                        
-                        d1, d2 = st.columns(2)
-                        with open("ldavis_img.png", "rb") as file:
-                            btn = d1.download_button(
-                                label="Download image",
-                                data=file,
-                                file_name="ldavis_img.png",
-                                mime="image/png"
-                                )
-                        
-                        #===download results===#
-                        resultf = pd.DataFrame(topics)
-                        #formatting
-                        resultf = resultf.transpose()
-                        resultf = resultf.drop([0])
-                        resultf = resultf.explode(list(range(len(resultf.columns))), ignore_index=False)
-                        
-                        resultcsv = resultf.to_csv().encode("utf-8")
-                        d2.download_button(
-                            label = "Download Results",
-                            data=resultcsv,
-                            file_name="results.csv",
-                            mime="text\csv",
-                            on_click="ignore")
-
-                    except NameError as f:
-                        st.warning('🖱️ Please click Submit')
-    
-            with tab2:
-                st.markdown('**Sievert, C., & Shirley, K. (2014). LDAvis: A method for visualizing and interpreting topics. Proceedings of the Workshop on Interactive Language Learning, Visualization, and Interfaces.** https://doi.org/10.3115/v1/w14-3110')
-    
-            with tab3:
-                st.markdown('**Chen, X., & Wang, H. (2019, January). Automated chat transcript analysis using topic modeling for library reference services. Proceedings of the Association for Information Science and Technology, 56(1), 368–371.** https://doi.org/10.1002/pra2.31')
-                st.markdown('**Joo, S., Ingram, E., & Cahill, M. (2021, December 15). Exploring Topics and Genres in Storytime Books: A Text Mining Approach. Evidence Based Library and Information Practice, 16(4), 41–62.** https://doi.org/10.18438/eblip29963')
-                st.markdown('**Lamba, M., & Madhusudhan, M. (2021, July 31). Topic Modeling. Text Mining for Information Professionals, 105–137.** https://doi.org/10.1007/978-3-030-85085-2_4')
-                st.markdown('**Lamba, M., & Madhusudhan, M. (2019, June 7). Mapping of topics in DESIDOC Journal of Library and Information Technology, India: a study. Scientometrics, 120(2), 477–505.** https://doi.org/10.1007/s11192-019-03137-5')
-         
-            with tab4:
-                st.subheader(':blue[pyLDA]', anchor=False)
-                st.button('Download image')
-                st.text("Click Download Image button.")
-                st.divider()
-                st.subheader(':blue[Downloading CSV Results]', anchor=False)
-                st.button("Download Results")
-                st.text("Click Download results button at bottom of page")
-
-         #===Biterm===
-        elif method == 'Biterm':            
-                 
-            #===optimize Biterm===
-            @st.cache_data(ttl=3600, show_spinner=False)
-            def biterm_topic(extype):
-                tokenized_abs = [t.split(' ') for t in topic_abs]
-
-                bigram = Phrases(tokenized_abs, min_count=xgram, threshold=opt_threshold)
-                trigram = Phrases(bigram[tokenized_abs], threshold=opt_threshold)
-                bigram_mod = Phraser(bigram)
-                trigram_mod = Phraser(trigram)
-            
-                topic_abs_ngram = [trigram_mod[bigram_mod[doc]] for doc in tokenized_abs]
-            
-                topic_abs_str = [' '.join(doc) for doc in topic_abs_ngram]
-
-                
-                X, vocabulary, vocab_dict = btm.get_words_freqs(topic_abs_str)
-                tf = np.array(X.sum(axis=0)).ravel()
-                docs_vec = btm.get_vectorized_docs(topic_abs, vocabulary)
-                docs_lens = list(map(len, docs_vec))
-                biterms = btm.get_biterms(docs_vec)
-                
-                model = btm.BTM(X, vocabulary, seed=btm_seed, T=num_topic, M=20, alpha=50/8, beta=0.01)
-                model.fit(biterms, iterations=btm_iterations)
-                
-                p_zd = model.transform(docs_vec)
-                coherence = model.coherence_
-                phi = tmp.get_phi(model)
-                topics_coords = tmp.prepare_coords(model)
-                totaltop = topics_coords.label.values.tolist()
-                perplexity = model.perplexity_
-                top_topics = model.df_words_topics_
-                
-                return topics_coords, phi, totaltop, perplexity, top_topics
-    
-            tab1, tab2, tab3, tab4 = st.tabs(["📈 Generate visualization", "📃 Reference", "📓 Recommended Reading", "⬇️ Download Help"])
-            with tab1:
-                try:
-                    with st.spinner('Performing computations. Please wait ...'): 
-                        topics_coords, phi, totaltop, perplexity, top_topics = biterm_topic(extype)            
-                        col1, col2 = st.columns([4,6])
-                      
-                        @st.cache_data(ttl=3600)
-                        def biterm_map(extype):
-                            btmvis_coords = tmp.plot_scatter_topics(topics_coords, size_col='size', label_col='label', topic=numvis)
-                            return btmvis_coords
-                                
-                        @st.cache_data(ttl=3600)
-                        def biterm_bar(extype):
-                            terms_probs = tmp.calc_terms_probs_ratio(phi, topic=numvis, lambda_=1)
-                            btmvis_probs = tmp.plot_terms(terms_probs, font_size=12)
-                            return btmvis_probs
-                                
-                        with col1:
-                            st.write('Perplexity score: ', perplexity)
-                            st.write('')
-                            numvis = st.selectbox(
-                                'Choose topic',
-                                (totaltop), on_change=reset_biterm)
-                            btmvis_coords = biterm_map(extype)
-                            st.altair_chart(btmvis_coords)
-                        with col2:
-                            btmvis_probs = biterm_bar(extype)
-                            st.altair_chart(btmvis_probs, use_container_width=True)
-    
-                        #===download results===#
-                        resultcsv = top_topics.to_csv().encode("utf-8")
-                        st.download_button(label = "Download Results", data=resultcsv, file_name="results.csv", mime="text\csv", on_click="ignore")
-
-                except ValueError as g:
-                    st.error('🙇‍♂️ Please raise the number of topics and click submit')
-                    
-                except NameError as f:
-                    st.warning('🖱️ Please click Submit')
-    
-            with tab2: 
-                st.markdown('**Yan, X., Guo, J., Lan, Y., & Cheng, X. (2013, May 13). A biterm topic model for short texts. Proceedings of the 22nd International Conference on World Wide Web.** https://doi.org/10.1145/2488388.2488514')
-            with tab3:
-                st.markdown('**Cai, M., Shah, N., Li, J., Chen, W. H., Cuomo, R. E., Obradovich, N., & Mackey, T. K. (2020, August 26). Identification and characterization of tweets related to the 2015 Indiana HIV outbreak: A retrospective infoveillance study. PLOS ONE, 15(8), e0235150.** https://doi.org/10.1371/journal.pone.0235150')
-                st.markdown('**Chen, Y., Dong, T., Ban, Q., & Li, Y. (2021). What Concerns Consumers about Hypertension? A Comparison between the Online Health Community and the Q&A Forum. International Journal of Computational Intelligence Systems, 14(1), 734.** https://doi.org/10.2991/ijcis.d.210203.002')
-                st.markdown('**George, Crissandra J., "AMBIGUOUS APPALACHIANNESS: A LINGUISTIC AND PERCEPTUAL INVESTIGATION INTO ARC-LABELED PENNSYLVANIA COUNTIES" (2022). Theses and Dissertations-- Linguistics. 48.** https://doi.org/10.13023/etd.2022.217')
-                st.markdown('**Li, J., Chen, W. H., Xu, Q., Shah, N., Kohler, J. C., & Mackey, T. K. (2020). Detection of self-reported experiences with corruption on twitter using unsupervised machine learning. Social Sciences & Humanities Open, 2(1), 100060.** https://doi.org/10.1016/j.ssaho.2020.100060')
-            with tab4:
-                st.subheader(':blue[Biterm]', anchor=False)
-                st.text("Click the three dots at the top right then select the desired format.")
-                st.markdown("![Downloading visualization](https://raw.githubusercontent.com/faizhalas/library-tools/main/images/download_biterm.jpg)")  
-                st.divider()
-                st.subheader(':blue[Downloading CSV Results]', anchor=False)
-                st.button("Download Results")
-                st.text("Click Download results button at bottom of page")
-
-
-         #===BERTopic===
-        elif method == 'BERTopic':
-            @st.cache_resource(ttl = 3600, show_spinner=False)
-            #@st.cache_data(ttl=3600, show_spinner=False)
-            def bertopic_vis(extype):
-                umap_model = UMAP(n_neighbors=bert_n_neighbors, n_components=bert_n_components, 
-                    min_dist=0.0, metric='cosine', random_state=bert_random_state)   
-                cluster_model = KMeans(n_clusters=num_topic)
-                if bert_embedding_model == 'all-MiniLM-L6-v2':
-                    model = SentenceTransformer('all-MiniLM-L6-v2')
-                    lang = 'en'
-                    embeddings = model.encode(topic_abs, show_progress_bar=True)
-                    
-                elif bert_embedding_model == 'en_core_web_sm':
-                    nlp = en_core_web_sm.load(exclude=['tagger', 'parser', 'ner', 'attribute_ruler', 'lemmatizer'])
-                    model = nlp
-                    lang = 'en'
-                    embeddings = np.array([nlp(text).vector for text in topic_abs])
-                    
-                elif bert_embedding_model == 'paraphrase-multilingual-MiniLM-L12-v2':
-                    model = SentenceTransformer('paraphrase-multilingual-MiniLM-L12-v2')
-                    lang = 'multilingual'
-                    embeddings = model.encode(topic_abs, show_progress_bar=True)
-
-                representation_model = ""
-
-                if fine_tuning:
-                    keybert = KeyBERTInspired()
-                    mmr = MaximalMarginalRelevance(diversity=0.3)
-                    representation_model = {
-                        "KeyBERT": keybert,
-                        "MMR": mmr,
-                    }
-                    if topic_labelling:
-                        if llm_provider == "OpenAI/gpt-4o":
-                            client = openai.OpenAI(api_key=api_key)
-                            representation_model = {
-                                "KeyBERT": keybert,
-                                "MMR": mmr,
-                                "test": OpenAI(client, model = "gpt-4o-mini", delay_in_seconds=10)
-                            }
-                        elif llm_provider == "Google/flan-t5":
-                            pipe = pipeline("text2text-generation", model = "google/flan-t5-base")
-                            clientmod = TextGeneration(pipe)
-                            representation_model = {
-                                "KeyBERT": keybert,
-                                "MMR": mmr,
-                                "test": clientmod
-                            }
-                        elif llm_provider == "LiquidAI/LFM2-350M":
-                            pipe = pipeline("text-generation", model = "LiquidAI/LFM2-350M")
-                            clientmod = TextGeneration(pipe)
-                            representation_model = {
-                                "KeyBERT": keybert,
-                                "MMR": mmr,
-                                "test": clientmod
-                            }
-
-                vectorizer_model = CountVectorizer(ngram_range=(1, xgram), stop_words='english')
-                topic_model = BERTopic(representation_model = representation_model, embedding_model=model, hdbscan_model=cluster_model, language=lang, umap_model=umap_model, vectorizer_model=vectorizer_model, top_n_words=bert_top_n_words)
-                topics, probs = topic_model.fit_transform(topic_abs, embeddings=embeddings)
-
-                if(fine_tuning and topic_labelling):
-                    generated_labels = [label[0][0].split("\n")[0] for label in topic_model.get_topics(full=True)["test"].values()]
-                    topic_model.set_topic_labels(generated_labels)
-
-                return topic_model, topics, probs, embeddings
-            
-            @st.cache_resource(ttl = 3600, show_spinner=False)
-            def Vis_Topics(extype):
-                fig1 = topic_model.visualize_topics(custom_labels = True)
-                return fig1
-            @st.cache_resource(ttl = 3600, show_spinner=False)
-            def Vis_Documents(extype):
-                fig2 = topic_model.visualize_document_datamap(topic_abs, embeddings=embeddings, custom_labels = True)
-                return fig2
-            @st.cache_resource(ttl = 3600, show_spinner=False)
-            def Vis_Hierarchy(extype):
-                fig3 = topic_model.visualize_hierarchy(top_n_topics=num_topic, custom_labels = True)
-                return fig3
-            @st.cache_resource(ttl = 3600, show_spinner=False)
-            def Vis_Heatmap(extype):
-                global topic_model
-                fig4 = topic_model.visualize_heatmap(n_clusters=num_topic-1, width=1000, height=1000, custom_labels = True)
-                return fig4
-            @st.cache_resource(ttl = 3600, show_spinner=False)
-            def Vis_Barchart(extype):
-                fig5 = topic_model.visualize_barchart(top_n_topics=num_topic, custom_labels = True)
-                return fig5
-           
-            tab1, tab2, tab3, tab4 = st.tabs(["📈 Generate visualization", "📃 Reference", "📓 Recommended Reading", "⬇️ Download Help"])
-            with tab1:
-                try:
-                    with st.spinner('Performing computations. Please wait ...'):
-                   
-                        topic_model, topics, probs, embeddings = bertopic_vis(extype)
-                        time.sleep(.5)
-                        st.toast('Visualize Topics', icon='🏃')
-                        fig1 = Vis_Topics(extype)
-                       
-                        time.sleep(.5)
-                        st.toast('Visualize Document', icon='🏃')
-                        fig2 = Vis_Documents(extype)
-                       
-                        time.sleep(.5)
-                        st.toast('Visualize Document Hierarchy', icon='🏃')
-                        fig3 = Vis_Hierarchy(extype)
-                       
-                        time.sleep(.5)
-                        st.toast('Visualize Topic Similarity', icon='🏃')
-                        fig4 = Vis_Heatmap(extype)
-                       
-                        time.sleep(.5)
-                        st.toast('Visualize Terms', icon='🏃')
-                        fig5 = Vis_Barchart(extype)
-                       
-                        bertab1, bertab2, bertab3, bertab4, bertab5 = st.tabs(["Visualize Topics", "Visualize Terms", "Visualize Documents",
-                                                                              "Visualize Document Hierarchy", "Visualize Topic Similarity"])
-                        
-                        with bertab1:
-                            st.plotly_chart(fig1, use_container_width=True)
-                        with bertab2:
-                            st.plotly_chart(fig5, use_container_width=True)
-                        with bertab3:
-                            st.plotly_chart(fig2, use_container_width=True)
-                        with bertab4:  
-                            st.plotly_chart(fig3, use_container_width=True)
-                        with bertab5:
-                            st.plotly_chart(fig4, use_container_width=True)
-                      
-                        #===download results===#
-                        results = topic_model.get_topic_info()
-                        resultf = pd.DataFrame(results)
-                        resultcsv = resultf.to_csv().encode("utf-8")
-                        st.download_button(
-                            label = "Download Results",
-                            data=resultcsv,
-                            file_name="results.csv",
-                            mime="text\csv",
-                            on_click="ignore",
-                        )
-
-                except ValueError:
-                    st.error('🙇‍♂️ Please raise the number of topics and click submit')
-
-
-                except NameError:
-                    st.warning('🖱️ Please click Submit')
-    
-            with tab2:
-                st.markdown('**Grootendorst, M. (2022). BERTopic: Neural topic modeling with a class-based TF-IDF procedure. arXiv preprint arXiv:2203.05794.** https://doi.org/10.48550/arXiv.2203.05794')
-              
-            with tab3:
-                st.markdown('**Jeet Rawat, A., Ghildiyal, S., & Dixit, A. K. (2022, December 1). Topic modelling of legal documents using NLP and bidirectional encoder representations from transformers. Indonesian Journal of Electrical Engineering and Computer Science, 28(3), 1749.** https://doi.org/10.11591/ijeecs.v28.i3.pp1749-1755')
-                st.markdown('**Yao, L. F., Ferawati, K., Liew, K., Wakamiya, S., & Aramaki, E. (2023, April 20). Disruptions in the Cystic Fibrosis Community’s Experiences and Concerns During the COVID-19 Pandemic: Topic Modeling and Time Series Analysis of Reddit Comments. Journal of Medical Internet Research, 25, e45249.** https://doi.org/10.2196/45249')
-
-            with tab4:
-                st.divider()
-                st.subheader(':blue[BERTopic]', anchor=False)
-                st.text("Click the camera icon on the top right menu")
-                st.markdown("![Downloading visualization](https://raw.githubusercontent.com/faizhalas/library-tools/main/images/download_bertopic.jpg)")
-                st.divider()
-                st.subheader(':blue[Downloading CSV Results]', anchor=False)
-                st.button("Download Results")
-                st.text("Click Download results button at bottom of page")
-
-    except:
-        st.error("Please ensure that your file is correct. Please contact us if you find that this is an error.", icon="🚨")
-        st.stop()
+#import module
+import streamlit as st
+import streamlit.components.v1 as components
+import pandas as pd
+import numpy as np
+import re
+import string
+import nltk
+nltk.download('wordnet')
+from nltk.stem import WordNetLemmatizer
+nltk.download('stopwords')
+from nltk.corpus import stopwords
+import gensim
+import gensim.corpora as corpora
+from gensim.corpora import Dictionary
+from gensim.models.coherencemodel import CoherenceModel
+from gensim.models.ldamodel import LdaModel
+from gensim.models import Phrases
+from gensim.models.phrases import Phraser
+from pprint import pprint
+import pickle
+import pyLDAvis
+import pyLDAvis.gensim_models as gensimvis
+from io import StringIO
+from ipywidgets.embed import embed_minimal_html
+from nltk.stem.snowball import SnowballStemmer
+from bertopic import BERTopic
+from bertopic.representation import KeyBERTInspired, MaximalMarginalRelevance, OpenAI, TextGeneration
+import plotly.express as px
+from sklearn.cluster import KMeans
+from sklearn.feature_extraction.text import CountVectorizer
+import bitermplus as btm
+import tmplot as tmp
+import tomotopy
+import sys
+import spacy
+import en_core_web_sm
+import pipeline
+from html2image import Html2Image
+from umap import UMAP
+import os
+import time
+import json
+from tools import sourceformat as sf
+import datamapplot
+from sentence_transformers import SentenceTransformer
+import openai
+from transformers import pipeline
+
+#===config===
+st.set_page_config(
+    page_title="Coconut",
+    page_icon="🥥",
+    layout="wide",
+    initial_sidebar_state="collapsed"
+)
+
+hide_streamlit_style = """
+            <style>
+            #MainMenu 
+            {visibility: hidden;}
+            footer {visibility: hidden;}
+            [data-testid="collapsedControl"] {display: none}
+            </style>
+            """
+st.markdown(hide_streamlit_style, unsafe_allow_html=True)
+
+with st.popover("🔗 Menu"):
+    st.page_link("https://www.coconut-libtool.com/", label="Home", icon="🏠")
+    st.page_link("pages/1 Scattertext.py", label="Scattertext", icon="1️⃣")
+    st.page_link("pages/2 Topic Modeling.py", label="Topic Modeling", icon="2️⃣")
+    st.page_link("pages/3 Bidirected Network.py", label="Bidirected Network", icon="3️⃣")
+    st.page_link("pages/4 Sunburst.py", label="Sunburst", icon="4️⃣")
+    st.page_link("pages/5 Burst Detection.py", label="Burst Detection", icon="5️⃣")
+    st.page_link("pages/6 Keywords Stem.py", label="Keywords Stem", icon="6️⃣")
+    st.page_link("pages/7 Sentiment Analysis.py", label="Sentiment Analysis", icon="7️⃣")
+    st.page_link("pages/8 Shifterator.py", label="Shifterator", icon="8️⃣")
+    st.page_link("pages/9 Summarization.py", label = "Summarization",icon ="9️⃣")
+    st.page_link("pages/10 WordCloud.py", label = "WordCloud", icon = "🔟")
+
+st.header("Topic Modeling", anchor=False)
+st.subheader('Put your file here...', anchor=False)
+
+#========unique id========
+@st.cache_resource(ttl=3600)
+def create_list():
+    l = [1, 2, 3]
+    return l
+
+l = create_list()
+first_list_value = l[0]
+l[0] = first_list_value + 1
+uID = str(l[0])
+
+@st.cache_data(ttl=3600)
+def get_ext(uploaded_file):
+    extype = uID+uploaded_file.name
+    return extype
+
+#===clear cache===
+
+def reset_biterm():
+    try:
+        biterm_map.clear()
+        biterm_bar.clear()
+    except NameError:
+        biterm_topic.clear()
+
+def reset_all():
+    st.cache_data.clear()
+
+#===avoiding deadlock===
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+        
+#===upload file===
+@st.cache_data(ttl=3600)
+def upload(file):
+    papers = pd.read_csv(uploaded_file)
+    if "About the data" in papers.columns[0]:
+        papers = sf.dim(papers)
+        col_dict = {'MeSH terms': 'Keywords',
+        'PubYear': 'Year',
+        'Times cited': 'Cited by',
+        'Publication Type': 'Document Type'
+        }
+        papers.rename(columns=col_dict, inplace=True)
+    
+    return papers
+
+@st.cache_data(ttl=3600)
+def conv_txt(extype):
+    if("PMID" in (uploaded_file.read()).decode()):
+        uploaded_file.seek(0)
+        papers = sf.medline(uploaded_file)
+        print(papers)
+        return papers
+    col_dict = {'TI': 'Title',
+            'SO': 'Source title',
+            'DE': 'Author Keywords',
+            'DT': 'Document Type',
+            'AB': 'Abstract',
+            'TC': 'Cited by',
+            'PY': 'Year',
+            'ID': 'Keywords Plus',
+            'rights_date_used': 'Year'}
+    uploaded_file.seek(0)
+    papers = pd.read_csv(uploaded_file, sep='\t')
+    if("htid" in papers.columns):
+        papers = sf.htrc(papers)
+    papers.rename(columns=col_dict, inplace=True)
+    print(papers)
+    return papers
+
+
+@st.cache_data(ttl=3600)
+def conv_json(extype):
+    col_dict={'title': 'title',
+    'rights_date_used': 'Year',
+    }
+
+    data = json.load(uploaded_file)
+    hathifile = data['gathers']
+    keywords = pd.DataFrame.from_records(hathifile)
+    
+    keywords = sf.htrc(keywords)
+    keywords.rename(columns=col_dict,inplace=True)
+    return keywords
+
+@st.cache_data(ttl=3600)
+def conv_pub(extype):
+    if (get_ext(extype)).endswith('.tar.gz'):
+        bytedata = extype.read()
+        keywords = sf.readPub(bytedata)
+    elif (get_ext(extype)).endswith('.xml'):
+        bytedata = extype.read()
+        keywords = sf.readxml(bytedata)
+    return keywords
+
+#===Read data===
+uploaded_file = st.file_uploader('', type=['csv', 'txt','json','tar.gz','xml'], on_change=reset_all)
+
+if uploaded_file is not None:
+    try:
+        extype = get_ext(uploaded_file)
+    
+        if extype.endswith('.csv'):
+             papers = upload(extype) 
+        elif extype.endswith('.txt'):
+             papers = conv_txt(extype)
+
+        elif extype.endswith('.json'):
+            papers = conv_json(extype)
+        elif extype.endswith('.tar.gz') or extype.endswith('.xml'):
+            papers = conv_pub(uploaded_file)
+
+        coldf = sorted(papers.select_dtypes(include=['object']).columns.tolist())
+            
+        c1, c2, c3 = st.columns([3,3,4])
+        method = c1.selectbox(
+                'Choose method',
+                ('Choose...', 'pyLDA', 'Biterm', 'BERTopic'))
+        ColCho = c2.selectbox('Choose column', (["Abstract","Title", "Abstract + Title"]))
+        num_cho = c3.number_input('Choose number of topics', min_value=2, max_value=30, value=5)
+
+        d1, d2 = st.columns([3,7])
+        xgram = d1.selectbox("N-grams", ("1", "2", "3"), on_change=reset_all)
+        xgram = int(xgram)
+        words_to_remove = d2.text_input("Remove specific words. Separate words by semicolons (;)", on_change=reset_all)
+    
+        rem_copyright = d1.toggle('Remove copyright statement', value=True, on_change=reset_all)
+        rem_punc = d2.toggle('Remove punctuation', value=True, on_change=reset_all)
+
+        #===advance settings===
+        with st.expander("🧮 Show advance settings"): 
+            t1, t2, t3, t4 = st.columns(4)
+            if method == 'pyLDA':
+                py_random_state = t1.number_input('Random state', min_value=0, max_value=None, step=1, help='Ensuring the reproducibility.')
+                py_chunksize = t2.number_input('Chunk size', value=100 , min_value=10, max_value=None, step=1, help='Number of documents to be used in each training chunk.')
+                opt_threshold = t3.number_input('Threshold (Gensim)', value=100 , min_value=1, max_value=None, step=1, help='Lower = More phrases. Higher = Fewer phrases.')
+                opt_relevance = t4.number_input('Lambda (λ)', value=0.6 , min_value=0.0, max_value=1.0, step=0.01, help='Lower = More unique. Higher = More frequent.')
+                
+                
+            elif method == 'Biterm':
+                btm_seed = t1.number_input('Random state seed', value=100 , min_value=1, max_value=None, step=1, help='Ensuring the reproducibility.')
+                btm_iterations = t2.number_input('Iterations number', value=20 , min_value=2, max_value=None, step=1, help='Number of iterations the model fitting process has gone through.')
+                opt_threshold = t3.number_input('Threshold (Gensim)', value=100 , min_value=1, max_value=None, step=1, help='Lower = More phrases. Higher = Fewer phrases.')
+                
+            elif method == 'BERTopic':
+                #u1, u2 = st.columns([5,5])
+                
+                bert_top_n_words = t1.number_input('top_n_words', value=5 , min_value=5, max_value=25, step=1, help='Number of words per topic.')
+                bert_random_state = t2.number_input('random_state', value=42 , min_value=1, max_value=None, step=1, help="Please be aware we currently can't do the reproducibility on Bertopic.")
+                bert_n_components = t3.number_input('n_components', value=5 , min_value=1, max_value=None, step=1, help='The dimensionality of the embeddings after reducing them.')
+                bert_n_neighbors = t4.number_input('n_neighbors', value=15 , min_value=1, max_value=None, step=1, help='The number of neighboring sample points used when making the manifold approximation.')
+                bert_embedding_model = st.radio(
+                    "embedding_model", 
+                    ["all-MiniLM-L6-v2", "paraphrase-multilingual-MiniLM-L12-v2", "en_core_web_sm"], index=0, horizontal=True, help= 'Select paraphrase-multilingual if your documents are in a language other than English or are multilingual.')
+                
+                fine_tuning = st.toggle("Use Fine-tuning")
+                if fine_tuning:
+                    topic_labelling = st.toggle("Automatic topic labelling")
+                    if topic_labelling:
+                        llm_provider = st.selectbox("Model",["OpenAI/gpt-4o","Google/flan-t5","LiquidAI/LFM2-350M"])
+                        if llm_provider == "OpenAI/gpt-4o":
+                            api_key = st.text_input("API Key")
+                
+            else:
+                st.write('Please choose your preferred method')
+
+        #===clean csv===
+        @st.cache_data(ttl=3600, show_spinner=False)
+        def clean_csv(extype):
+            if (ColCho=="Abstract + Title"):
+                papers["Abstract + Title"] = papers["Title"] + " " + papers["Abstract"]
+                st.write(papers["Abstract + Title"])
+
+            paper = papers.dropna(subset=[ColCho])
+                     
+            #===mapping===
+            paper['Abstract_pre'] = paper[ColCho].map(lambda x: x.lower())
+            if rem_punc:
+                paper['Abstract_pre'] = paper['Abstract_pre'].map(
+                    lambda x: re.sub(f"[{re.escape(string.punctuation)}]", " ", x)
+                ).map(lambda x: re.sub(r"\s+", " ", x).strip())
+                paper['Abstract_pre'] = paper['Abstract_pre'].str.replace('[\u2018\u2019\u201c\u201d]', '', regex=True)
+            if rem_copyright:  
+                paper['Abstract_pre'] = paper['Abstract_pre'].map(lambda x: re.sub('©.*', '', x))
+            
+            #===stopword removal===
+            stop = stopwords.words('english')
+            paper['Abstract_stop'] = paper['Abstract_pre'].apply(lambda x: ' '.join([word for word in x.split() if word not in (stop)]))
+              
+            #===lemmatize===
+            lemmatizer = WordNetLemmatizer()
+            
+            @st.cache_data(ttl=3600)
+            def lemmatize_words(text):
+                words = text.split()
+                words = [lemmatizer.lemmatize(word) for word in words]
+                return ' '.join(words)
+            paper['Abstract_lem'] = paper['Abstract_stop'].apply(lemmatize_words)
+        
+            words_rmv = [word.strip() for word in words_to_remove.split(";")]
+            remove_dict = {word: None for word in words_rmv}
+            
+            @st.cache_data(ttl=3600)
+            def remove_words(text):
+                 words = text.split()
+                 cleaned_words = [word for word in words if word not in remove_dict]
+                 return ' '.join(cleaned_words) 
+            paper['Abstract_lem'] = paper['Abstract_lem'].map(remove_words)
+             
+            topic_abs = paper.Abstract_lem.values.tolist()
+            return topic_abs, paper
+    
+        topic_abs, paper=clean_csv(extype) 
+                 
+        if st.button("Submit", on_click=reset_all):
+            num_topic = num_cho  
+    
+        if method == 'BERTopic':
+            st.info('BERTopic is an expensive process when dealing with a large volume of text with our existing resources. Please kindly wait until the visualization appears.', icon="ℹ️")
+               
+        #===topic===
+        if method == 'Choose...':
+            st.write('')
+    
+        elif method == 'pyLDA':       
+            tab1, tab2, tab3, tab4 = st.tabs(["📈 Generate visualization", "📃 Reference", "📓 Recommended Reading", "⬇️ Download Help"])
+    
+            with tab1:
+            #===visualization===
+                @st.cache_data(ttl=3600, show_spinner=False)
+                def pylda(extype):
+                    topic_abs_LDA = [t.split(' ') for t in topic_abs]
+
+                    bigram = Phrases(topic_abs_LDA, min_count=xgram, threshold=opt_threshold)
+                    trigram = Phrases(bigram[topic_abs_LDA], threshold=opt_threshold)
+                    bigram_mod = Phraser(bigram)
+                    trigram_mod = Phraser(trigram)
+                    
+                    topic_abs_LDA = [trigram_mod[bigram_mod[doc]] for doc in topic_abs_LDA]
+
+                    id2word = Dictionary(topic_abs_LDA)
+                    corpus = [id2word.doc2bow(text) for text in topic_abs_LDA]
+                    #===LDA===
+                    lda_model = LdaModel(corpus=corpus,
+                                id2word=id2word,
+                                num_topics=num_topic, 
+                                random_state=py_random_state,
+                                chunksize=py_chunksize,
+                                alpha='auto',
+                                gamma_threshold=opt_relevance,
+                                per_word_topics=False)
+                    pprint(lda_model.print_topics())
+                    doc_lda = lda_model[corpus]
+                    topics = lda_model.show_topics(num_words = 30,formatted=False)
+
+                    #===visualization===
+                    coherence_model_lda = CoherenceModel(model=lda_model, texts=topic_abs_LDA, dictionary=id2word, coherence='c_v')
+                    coherence_lda = coherence_model_lda.get_coherence()
+                    vis = pyLDAvis.gensim_models.prepare(lda_model, corpus, id2word)
+                    py_lda_vis_html = pyLDAvis.prepared_data_to_html(vis)
+                    return py_lda_vis_html, coherence_lda, vis, topics
+                       
+                with st.spinner('Performing computations. Please wait ...'):
+                    try:
+                        py_lda_vis_html, coherence_lda, vis, topics = pylda(extype)
+                        st.write('Coherence score: ', coherence_lda)
+                        components.html(py_lda_vis_html, width=1500, height=800)
+                        st.markdown('Copyright (c) 2015, Ben Mabey. https://github.com/bmabey/pyLDAvis')
+                           
+                        @st.cache_data(ttl=3600, show_spinner=False)
+                        def img_lda(vis):
+                            pyLDAvis.save_html(vis, 'output.html')
+                            hti = Html2Image()
+                            hti.browser.flags = ['--default-background-color=ffffff', '--hide-scrollbars']
+                            hti.browser.use_new_headless = None
+                            css = "body {background: white;}"
+                            hti.screenshot( 
+                                other_file='output.html', css_str=css, size=(1500, 800),
+                                save_as='ldavis_img.png'
+                            )
+
+                        img_lda(vis)
+                        
+                        d1, d2 = st.columns(2)
+                        with open("ldavis_img.png", "rb") as file:
+                            btn = d1.download_button(
+                                label="Download image",
+                                data=file,
+                                file_name="ldavis_img.png",
+                                mime="image/png"
+                                )
+                        
+                        #===download results===#
+                        resultf = pd.DataFrame(topics)
+                        #formatting
+                        resultf = resultf.transpose()
+                        resultf = resultf.drop([0])
+                        resultf = resultf.explode(list(range(len(resultf.columns))), ignore_index=False)
+                        
+                        resultcsv = resultf.to_csv().encode("utf-8")
+                        d2.download_button(
+                            label = "Download Results",
+                            data=resultcsv,
+                            file_name="results.csv",
+                            mime="text\csv",
+                            on_click="ignore")
+
+                    except NameError as f:
+                        st.warning('🖱️ Please click Submit')
+    
+            with tab2:
+                st.markdown('**Sievert, C., & Shirley, K. (2014). LDAvis: A method for visualizing and interpreting topics. Proceedings of the Workshop on Interactive Language Learning, Visualization, and Interfaces.** https://doi.org/10.3115/v1/w14-3110')
+    
+            with tab3:
+                st.markdown('**Chen, X., & Wang, H. (2019, January). Automated chat transcript analysis using topic modeling for library reference services. Proceedings of the Association for Information Science and Technology, 56(1), 368–371.** https://doi.org/10.1002/pra2.31')
+                st.markdown('**Joo, S., Ingram, E., & Cahill, M. (2021, December 15). Exploring Topics and Genres in Storytime Books: A Text Mining Approach. Evidence Based Library and Information Practice, 16(4), 41–62.** https://doi.org/10.18438/eblip29963')
+                st.markdown('**Lamba, M., & Madhusudhan, M. (2021, July 31). Topic Modeling. Text Mining for Information Professionals, 105–137.** https://doi.org/10.1007/978-3-030-85085-2_4')
+                st.markdown('**Lamba, M., & Madhusudhan, M. (2019, June 7). Mapping of topics in DESIDOC Journal of Library and Information Technology, India: a study. Scientometrics, 120(2), 477–505.** https://doi.org/10.1007/s11192-019-03137-5')
+         
+            with tab4:
+                st.subheader(':blue[pyLDA]', anchor=False)
+                st.button('Download image')
+                st.text("Click Download Image button.")
+                st.divider()
+                st.subheader(':blue[Downloading CSV Results]', anchor=False)
+                st.button("Download Results")
+                st.text("Click Download results button at bottom of page")
+
+         #===Biterm===
+        elif method == 'Biterm':            
+                 
+            #===optimize Biterm===
+            @st.cache_data(ttl=3600, show_spinner=False)
+            def biterm_topic(extype):
+                tokenized_abs = [t.split(' ') for t in topic_abs]
+
+                bigram = Phrases(tokenized_abs, min_count=xgram, threshold=opt_threshold)
+                trigram = Phrases(bigram[tokenized_abs], threshold=opt_threshold)
+                bigram_mod = Phraser(bigram)
+                trigram_mod = Phraser(trigram)
+            
+                topic_abs_ngram = [trigram_mod[bigram_mod[doc]] for doc in tokenized_abs]
+            
+                topic_abs_str = [' '.join(doc) for doc in topic_abs_ngram]
+
+                
+                X, vocabulary, vocab_dict = btm.get_words_freqs(topic_abs_str)
+                tf = np.array(X.sum(axis=0)).ravel()
+                docs_vec = btm.get_vectorized_docs(topic_abs, vocabulary)
+                docs_lens = list(map(len, docs_vec))
+                biterms = btm.get_biterms(docs_vec)
+                
+                model = btm.BTM(X, vocabulary, seed=btm_seed, T=num_topic, M=20, alpha=50/8, beta=0.01)
+                model.fit(biterms, iterations=btm_iterations)
+                
+                p_zd = model.transform(docs_vec)
+                coherence = model.coherence_
+                phi = tmp.get_phi(model)
+                topics_coords = tmp.prepare_coords(model)
+                totaltop = topics_coords.label.values.tolist()
+                perplexity = model.perplexity_
+                top_topics = model.df_words_topics_
+                
+                return topics_coords, phi, totaltop, perplexity, top_topics
+    
+            tab1, tab2, tab3, tab4 = st.tabs(["📈 Generate visualization", "📃 Reference", "📓 Recommended Reading", "⬇️ Download Help"])
+            with tab1:
+                try:
+                    with st.spinner('Performing computations. Please wait ...'): 
+                        topics_coords, phi, totaltop, perplexity, top_topics = biterm_topic(extype)            
+                        col1, col2 = st.columns([4,6])
+                      
+                        @st.cache_data(ttl=3600)
+                        def biterm_map(extype):
+                            btmvis_coords = tmp.plot_scatter_topics(topics_coords, size_col='size', label_col='label', topic=numvis)
+                            return btmvis_coords
+                                
+                        @st.cache_data(ttl=3600)
+                        def biterm_bar(extype):
+                            terms_probs = tmp.calc_terms_probs_ratio(phi, topic=numvis, lambda_=1)
+                            btmvis_probs = tmp.plot_terms(terms_probs, font_size=12)
+                            return btmvis_probs
+                                
+                        with col1:
+                            st.write('Perplexity score: ', perplexity)
+                            st.write('')
+                            numvis = st.selectbox(
+                                'Choose topic',
+                                (totaltop), on_change=reset_biterm)
+                            btmvis_coords = biterm_map(extype)
+                            st.altair_chart(btmvis_coords)
+                        with col2:
+                            btmvis_probs = biterm_bar(extype)
+                            st.altair_chart(btmvis_probs, use_container_width=True)
+    
+                        #===download results===#
+                        resultcsv = top_topics.to_csv().encode("utf-8")
+                        st.download_button(label = "Download Results", data=resultcsv, file_name="results.csv", mime="text\csv", on_click="ignore")
+
+                except ValueError as g:
+                    st.error('🙇‍♂️ Please raise the number of topics and click submit')
+                    
+                except NameError as f:
+                    st.warning('🖱️ Please click Submit')
+    
+            with tab2: 
+                st.markdown('**Yan, X., Guo, J., Lan, Y., & Cheng, X. (2013, May 13). A biterm topic model for short texts. Proceedings of the 22nd International Conference on World Wide Web.** https://doi.org/10.1145/2488388.2488514')
+            with tab3:
+                st.markdown('**Cai, M., Shah, N., Li, J., Chen, W. H., Cuomo, R. E., Obradovich, N., & Mackey, T. K. (2020, August 26). Identification and characterization of tweets related to the 2015 Indiana HIV outbreak: A retrospective infoveillance study. PLOS ONE, 15(8), e0235150.** https://doi.org/10.1371/journal.pone.0235150')
+                st.markdown('**Chen, Y., Dong, T., Ban, Q., & Li, Y. (2021). What Concerns Consumers about Hypertension? A Comparison between the Online Health Community and the Q&A Forum. International Journal of Computational Intelligence Systems, 14(1), 734.** https://doi.org/10.2991/ijcis.d.210203.002')
+                st.markdown('**George, Crissandra J., "AMBIGUOUS APPALACHIANNESS: A LINGUISTIC AND PERCEPTUAL INVESTIGATION INTO ARC-LABELED PENNSYLVANIA COUNTIES" (2022). Theses and Dissertations-- Linguistics. 48.** https://doi.org/10.13023/etd.2022.217')
+                st.markdown('**Li, J., Chen, W. H., Xu, Q., Shah, N., Kohler, J. C., & Mackey, T. K. (2020). Detection of self-reported experiences with corruption on twitter using unsupervised machine learning. Social Sciences & Humanities Open, 2(1), 100060.** https://doi.org/10.1016/j.ssaho.2020.100060')
+            with tab4:
+                st.subheader(':blue[Biterm]', anchor=False)
+                st.text("Click the three dots at the top right then select the desired format.")
+                st.markdown("![Downloading visualization](https://raw.githubusercontent.com/faizhalas/library-tools/main/images/download_biterm.jpg)")  
+                st.divider()
+                st.subheader(':blue[Downloading CSV Results]', anchor=False)
+                st.button("Download Results")
+                st.text("Click Download results button at bottom of page")
+
+
+         #===BERTopic===
+        elif method == 'BERTopic':
+            @st.cache_resource(ttl = 3600, show_spinner=False)
+            #@st.cache_data(ttl=3600, show_spinner=False)
+            def bertopic_vis(extype):
+                umap_model = UMAP(n_neighbors=bert_n_neighbors, n_components=bert_n_components, 
+                    min_dist=0.0, metric='cosine', random_state=bert_random_state)   
+                cluster_model = KMeans(n_clusters=num_topic)
+                if bert_embedding_model == 'all-MiniLM-L6-v2':
+                    model = SentenceTransformer('all-MiniLM-L6-v2')
+                    lang = 'en'
+                    embeddings = model.encode(topic_abs, show_progress_bar=True)
+                    
+                elif bert_embedding_model == 'en_core_web_sm':
+                    nlp = en_core_web_sm.load(exclude=['tagger', 'parser', 'ner', 'attribute_ruler', 'lemmatizer'])
+                    model = nlp
+                    lang = 'en'
+                    embeddings = np.array([nlp(text).vector for text in topic_abs])
+                    
+                elif bert_embedding_model == 'paraphrase-multilingual-MiniLM-L12-v2':
+                    model = SentenceTransformer('paraphrase-multilingual-MiniLM-L12-v2')
+                    lang = 'multilingual'
+                    embeddings = model.encode(topic_abs, show_progress_bar=True)
+
+                representation_model = ""
+
+                if fine_tuning:
+                    keybert = KeyBERTInspired()
+                    mmr = MaximalMarginalRelevance(diversity=0.3)
+                    representation_model = {
+                        "KeyBERT": keybert,
+                        "MMR": mmr,
+                    }
+                    if topic_labelling:
+                        if llm_provider == "OpenAI/gpt-4o":
+                            client = openai.OpenAI(api_key=api_key)
+                            representation_model = {
+                                "KeyBERT": keybert,
+                                "MMR": mmr,
+                                "test": OpenAI(client, model = "gpt-4o-mini", delay_in_seconds=10)
+                            }
+                        elif llm_provider == "Google/flan-t5":
+                            pipe = pipeline("text2text-generation", model = "google/flan-t5-base")
+                            clientmod = TextGeneration(pipe)
+                            representation_model = {
+                                "KeyBERT": keybert,
+                                "MMR": mmr,
+                                "test": clientmod
+                            }
+                        elif llm_provider == "LiquidAI/LFM2-350M":
+                            pipe = pipeline("text-generation", model = "LiquidAI/LFM2-350M")
+                            clientmod = TextGeneration(pipe)
+                            representation_model = {
+                                "KeyBERT": keybert,
+                                "MMR": mmr,
+                                "test": clientmod
+                            }
+
+                vectorizer_model = CountVectorizer(ngram_range=(1, xgram), stop_words='english')
+                topic_model = BERTopic(representation_model = representation_model, embedding_model=model, hdbscan_model=cluster_model, language=lang, umap_model=umap_model, vectorizer_model=vectorizer_model, top_n_words=bert_top_n_words)
+                topics, probs = topic_model.fit_transform(topic_abs, embeddings=embeddings)
+
+                if(fine_tuning and topic_labelling):
+                    generated_labels = [label[0][0].split("\n")[0] for label in topic_model.get_topics(full=True)["test"].values()]
+                    topic_model.set_topic_labels(generated_labels)
+
+                return topic_model, topics, probs, embeddings
+            
+            @st.cache_resource(ttl = 3600, show_spinner=False)
+            def Vis_Topics(extype):
+                fig1 = topic_model.visualize_topics(custom_labels = True)
+                return fig1
+            @st.cache_resource(ttl = 3600, show_spinner=False)
+            def Vis_Documents(extype):
+                fig2 = topic_model.visualize_document_datamap(topic_abs, embeddings=embeddings, custom_labels = True)
+                return fig2
+            @st.cache_resource(ttl = 3600, show_spinner=False)
+            def Vis_Hierarchy(extype):
+                fig3 = topic_model.visualize_hierarchy(top_n_topics=num_topic, custom_labels = True)
+                return fig3
+            @st.cache_resource(ttl = 3600, show_spinner=False)
+            def Vis_Heatmap(extype):
+                global topic_model
+                fig4 = topic_model.visualize_heatmap(n_clusters=num_topic-1, width=1000, height=1000, custom_labels = True)
+                return fig4
+            @st.cache_resource(ttl = 3600, show_spinner=False)
+            def Vis_Barchart(extype):
+                fig5 = topic_model.visualize_barchart(top_n_topics=num_topic, custom_labels = True)
+                return fig5
+           
+            tab1, tab2, tab3, tab4 = st.tabs(["📈 Generate visualization", "📃 Reference", "📓 Recommended Reading", "⬇️ Download Help"])
+            with tab1:
+                try:
+                    with st.spinner('Performing computations. Please wait ...'):
+                   
+                        topic_model, topics, probs, embeddings = bertopic_vis(extype)
+                        time.sleep(.5)
+                        st.toast('Visualize Topics', icon='🏃')
+                        fig1 = Vis_Topics(extype)
+                       
+                        time.sleep(.5)
+                        st.toast('Visualize Document', icon='🏃')
+                        fig2 = Vis_Documents(extype)
+                       
+                        time.sleep(.5)
+                        st.toast('Visualize Document Hierarchy', icon='🏃')
+                        fig3 = Vis_Hierarchy(extype)
+                       
+                        time.sleep(.5)
+                        st.toast('Visualize Topic Similarity', icon='🏃')
+                        fig4 = Vis_Heatmap(extype)
+                       
+                        time.sleep(.5)
+                        st.toast('Visualize Terms', icon='🏃')
+                        fig5 = Vis_Barchart(extype)
+                       
+                        bertab1, bertab2, bertab3, bertab4, bertab5 = st.tabs(["Visualize Topics", "Visualize Terms", "Visualize Documents",
+                                                                              "Visualize Document Hierarchy", "Visualize Topic Similarity"])
+                        
+                        with bertab1:
+                            st.plotly_chart(fig1, use_container_width=True)
+                        with bertab2:
+                            st.plotly_chart(fig5, use_container_width=True)
+                        with bertab3:
+                            st.plotly_chart(fig2, use_container_width=True)
+                        with bertab4:  
+                            st.plotly_chart(fig3, use_container_width=True)
+                        with bertab5:
+                            st.plotly_chart(fig4, use_container_width=True)
+                      
+                        #===download results===#
+                        results = topic_model.get_topic_info()
+                        resultf = pd.DataFrame(results)
+                        resultcsv = resultf.to_csv().encode("utf-8")
+                        st.download_button(
+                            label = "Download Results",
+                            data=resultcsv,
+                            file_name="results.csv",
+                            mime="text\csv",
+                            on_click="ignore",
+                        )
+
+                except ValueError:
+                    st.error('🙇‍♂️ Please raise the number of topics and click submit')
+
+
+                except NameError:
+                    st.warning('🖱️ Please click Submit')
+    
+            with tab2:
+                st.markdown('**Grootendorst, M. (2022). BERTopic: Neural topic modeling with a class-based TF-IDF procedure. arXiv preprint arXiv:2203.05794.** https://doi.org/10.48550/arXiv.2203.05794')
+              
+            with tab3:
+                st.markdown('**Jeet Rawat, A., Ghildiyal, S., & Dixit, A. K. (2022, December 1). Topic modelling of legal documents using NLP and bidirectional encoder representations from transformers. Indonesian Journal of Electrical Engineering and Computer Science, 28(3), 1749.** https://doi.org/10.11591/ijeecs.v28.i3.pp1749-1755')
+                st.markdown('**Yao, L. F., Ferawati, K., Liew, K., Wakamiya, S., & Aramaki, E. (2023, April 20). Disruptions in the Cystic Fibrosis Community’s Experiences and Concerns During the COVID-19 Pandemic: Topic Modeling and Time Series Analysis of Reddit Comments. Journal of Medical Internet Research, 25, e45249.** https://doi.org/10.2196/45249')
+
+            with tab4:
+                st.divider()
+                st.subheader(':blue[BERTopic]', anchor=False)
+                st.text("Click the camera icon on the top right menu")
+                st.markdown("![Downloading visualization](https://raw.githubusercontent.com/faizhalas/library-tools/main/images/download_bertopic.jpg)")
+                st.divider()
+                st.subheader(':blue[Downloading CSV Results]', anchor=False)
+                st.button("Download Results", on_click="ignore")
+                st.text("Click Download results button at bottom of page")
+
+    except:
+        st.error("Please ensure that your file is correct. Please contact us if you find that this is an error.", icon="🚨")
+        st.stop()

pages/3 Bidirected Network.py

@@ -235,16 +235,16 @@ if uploaded_file is not None:
         col1, col2, col3 = st.columns(3)
         with col1:
             supp = st.slider(
-                'Select value of Support',
-                0.001, 1.000, (0.010), on_change=reset_all)
+                'Support',
+                0.001, 1.000, (0.010), on_change=reset_all, help='Frequency of occurrence of keywords in a set of documents')
         with col2:
             conf = st.slider(
-                'Select value of Confidence',
-                0.001, 1.000, (0.050), on_change=reset_all)
+                'Confidence',
+                0.001, 1.000, (0.050), on_change=reset_all, help='Presence of keywords in documents that included the antecedents')
         with col3:
             maxlen = st.slider(
                 'Maximum length of the itemsets generated',
-                2, 8, (2), on_change=reset_all)
+                2, 8, (2), on_change=reset_all, help='')
     
         tab1, tab2, tab3, tab4 = st.tabs(["📈 Result & Generate visualization", "📃 Reference", "📓 Recommended Reading", "⬇️ Download Help"])
         
@@ -278,7 +278,7 @@ if uploaded_file is not None:
                 st.error('Please lower your value.', icon="🚨")
             else:
                 restab = arm_table(extype)
-                restab = st.data_editor(restab, use_container_width=True)
+                restab = st.data_editor(restab, use_container_width=True, hide_index=True)
                 res = restab[restab['Show'] == True] 
                        
                  #===visualize===
@@ -400,7 +400,7 @@ if uploaded_file is not None:
             st.markdown("![Downloading graph](https://raw.githubusercontent.com/faizhalas/library-tools/main/images/download_bidirected.jpg)")     
             st.subheader("Download table as CSV")
             st.text("Hover cursor over table, and click download arrow")
-            st.image("images/tablenetwork.png")
+            st.markdown("![Downloading table](https://raw.githubusercontent.com/faizhalas/library-tools/refs/heads/main/images/tablenetwork.png)")
             
     except:
         st.error("Please ensure that your file is correct. Please contact us if you find that this is an error.", icon="🚨")

pages/4 Sunburst.py

@@ -4,7 +4,6 @@ import pandas as pd
 import plotly.express as px
 import numpy as np
 import sys
-import json
 from tools import sourceformat as sf
 
 
@@ -94,6 +93,7 @@ def conv_txt(extype):
     print(papers)
     return papers
 
+
 @st.cache_data(ttl=3600)
 def conv_json(extype):
     col_dict={'title': 'title',
@@ -101,11 +101,7 @@ def conv_json(extype):
     'content_provider_code': 'Document Type',
     'Keywords':'Source title'
     }
-
-    data = json.load(uploaded_file)
-    hathifile = data['gathers']
-    keywords = pd.DataFrame.from_records(hathifile)
-
+    keywords = pd.read_json(uploaded_file)
     keywords = sf.htrc(keywords)
     keywords['Cited by'] = keywords.groupby(['Keywords'])['Keywords'].transform('size')
     keywords.rename(columns=col_dict,inplace=True)
@@ -146,37 +142,46 @@ if uploaded_file is not None:
             MIN1 = int(papers['Cited by'].min())
             MAX1 = int(papers['Cited by'].max())
             GAP = MAX - MIN
-            return papers, MIN, MAX, GAP, MIN1, MAX1
+            unique_stitle = set()
+            unique_stitle.update(papers['Source title'].dropna())
+            list_stitle = sorted(list(unique_stitle))
+            return papers, MIN, MAX, GAP, MIN1, MAX1, list_stitle
 
         tab1, tab2, tab3 = st.tabs(["📈 Generate visualization", "📓 Recommended Reading", "⬇️ Download Help"])
         
         with tab1:    
             #===sunburst===
             try:
-                papers, MIN, MAX, GAP, MIN1, MAX1 = get_minmax(extype)
+                papers, MIN, MAX, GAP, MIN1, MAX1, list_stitle = get_minmax(extype)
             except KeyError:
                 st.error('Error: Please check again your columns.')
                 sys.exit(1)
+
+            stitle = st.selectbox('Focus on', (list_stitle), index=None, on_change=reset_all)
             
             if (GAP != 0):
-                YEAR = st.slider('Year', min_value=MIN, max_value=MAX, value=(MIN, MAX), on_change=reset_all)
-                KEYLIM = st.slider('Cited By Count',min_value = MIN1, max_value = MAX1, value = (MIN1,MAX1), on_change=reset_all)
+                YEAR = st.slider('Year', min_value=MIN, max_value=MAX, value=(MIN, MAX))
+                KEYLIM = st.slider('Cited By Count',min_value = MIN1, max_value = MAX1, value = (MIN1,MAX1))
                 with st.expander("Filtering setings"):
-                    invert_keys = st.toggle("Invert keys", on_change=reset_all)
-                    filtered_keys = st.text_input("Filter words in source, seperate with semicolon (;)", value = "", on_change = reset_all)
+                    invert_keys = st.toggle("Invert keys")
+                    filtered_keys = st.text_input("Filter words in source, seperate with semicolon (;)", value = "\n")
                     keylist = filtered_keys.split(";") 
                     select_col = st.selectbox("Column to filter from", (list(papers)))
             else:
-                st.write('You only have data in ', (MAX))
+                col1.write('You only have data in ', (MAX))
                 YEAR = (MIN, MAX)
-                KEYLIM = (MIN1,MAX1)
+                KEYLIM = col2.slider('Cited By Count',min_value = MIN1, max_value = MAX1, value = (MIN1,MAX1), on_change=reset_all)
+                
             @st.cache_data(ttl=3600)
             def listyear(extype):
                 global papers
                 years = list(range(YEAR[0],YEAR[1]+1))
                 cited = list(range(KEYLIM[0],KEYLIM[1]+1))
+                if stitle:
+                    papers = papers[papers['Source title'].str.contains(stitle, case=False, na=False)]
                 papers = papers.loc[papers['Year'].isin(years)]
                 papers = papers.loc[papers['Cited by'].isin(cited)]
+                papers['Cited by'] = papers['Cited by'].fillna(0)
                 return years, papers
             
             @st.cache_data(ttl=3600)
@@ -186,9 +191,9 @@ if uploaded_file is not None:
 
                 #filtering
                 if(invert_keys):
-                    data = data[data[select_col].isin(keylist)]
+                    data = data[data[select_col].str.contains('|'.join(keylist), na=False)]
                 else:
-                    data = data[~data[select_col].isin(keylist)]
+                    data = data[~data[select_col].str.contains('|'.join(keylist), na=False)]
 
                 vis = pd.DataFrame()
                 vis[['doctype','source','citby','year']] = data[['Document Type','Source title','Cited by','Year']]
@@ -200,14 +205,14 @@ if uploaded_file is not None:
                               color_continuous_scale='RdBu',
                               color_continuous_midpoint=np.average(viz['cited by'], weights=viz['total docs']))
                 fig.update_layout(height=800, width=1200)
-                return fig, viz
+                return fig
             
             years, papers = listyear(extype)
     
             
             if {'Document Type','Source title','Cited by','Year'}.issubset(papers.columns):
               
-                if st.button("Submit"):
+                if st.button("Submit", on_click = reset_all):
                     fig, viz = vis_sunbrust(extype)
                     st.plotly_chart(fig, height=800, width=1200) #use_container_width=True)
                     st.dataframe(viz)
@@ -222,6 +227,7 @@ if uploaded_file is not None:
         with tab3:
             st.text("Click the camera icon on the top right menu (you may need to hover your cursor within the visualization)")
             st.markdown("![Downloading visualization](https://raw.githubusercontent.com/faizhalas/library-tools/main/images/download_bertopic.jpg)")
-    except:
+    except Exception as e:
+        st.write(e)
         st.error("Please ensure that your file is correct. Please contact us if you find that this is an error.", icon="🚨")
         st.stop()

pages/5 Burst Detection.py

@@ -364,84 +364,110 @@ def scattervis(bursts, freq_data, top_n):
         autosize=False
     )
                     
-    fig.write_image("scatter_plot.png")
-    st.image("scatter_plot.png")
-    pio.write_image(fig, 'result.png', scale=4)  
+    fig.write_image("scatter_plot.png", width=plot_width, height=plot_height)
+    st.image("scatter_plot.png", use_column_width=False)
+        
+    pio.write_image(fig, 'result.png', width=plot_width, height=plot_height, scale=4)  
 
 @st.cache_data(ttl=3600)
-def linegraph(bursts, freq_data):
-    fig = make_subplots(rows=num_rows, cols=2, subplot_titles=freq_data.columns[:top_n])
-    
+@st.cache_data(ttl=3600)
+def linegraph(bursts, freq_data, top_n, running_total=""):
+    num_rows = (top_n + 1) // 2   # 2 columns layout
+
+    # --- X spacing: each year gets a slot of width 10 (=> ±5 padding) ---
+    years = list(freq_data.index)
+    spacing = 100
+    padding = 200
+    x_positions = np.arange(len(years)) * spacing      # 0,10,20,...
+    tickvals = x_positions
+    ticktext = [str(y) for y in years]
+
+    fig = make_subplots(
+        rows=num_rows,
+        cols=2,
+        subplot_titles=freq_data.columns[:top_n]
+    )
+
     row, col = 1, 1
     for i, column in enumerate(freq_data.columns[:top_n]):
+        # main line (x mapped to spaced positions)
         fig.add_trace(go.Scatter(
-            x=freq_data.index, y=freq_data[column], mode='lines+markers+text', name=column,
+            x=x_positions,
+            y=freq_data[column].to_numpy(),
+            mode='lines+markers+text',
+            name=column,
             line_shape='linear',
             hoverinfo='text',
-            hovertext=[f"Year: {index}<br>Frequency: {freq}" for index, freq in zip(freq_data.index, freq_data[column])],
-            #text=freq_data[column],
+            hovertext=[f"Year: {yr}<br>Frequency: {freq}"
+                       for yr, freq in zip(years, freq_data[column])],
+            text=freq_data[column],
             textposition='top center'
         ), row=row, col=col)
-        # Add area charts
+
+        # bursts shading + annotation
         for _, row_data in bursts[bursts['label'] == column].iterrows():
-            x_values = freq_data.index[row_data['begin']:row_data['end']+1]
-            y_values = freq_data[column][row_data['begin']:row_data['end']+1]
-            #middle_y = sum(y_values) / len(y_values)
-            y_post = min(freq_data[column]) + 1 if running_total == "Running total" else sum(y_values) / len(y_values)
-            x_offset = 0.1
-                        
-            # Add area chart
+            # slice by positional indices (begin/end are positions)
+            x_vals = x_positions[row_data['begin']:row_data['end'] + 1]
+            y_vals = freq_data[column].iloc[row_data['begin']:row_data['end'] + 1].to_numpy()
+
+            # area under the line during burst
             fig.add_trace(go.Scatter(
-                x=x_values,
-                y=y_values,
-                fill='tozeroy', mode='lines', fillcolor='rgba(0,100,80,0.2)',
+                x=x_vals,
+                y=y_vals,
+                fill='tozeroy',
+                mode='lines',
+                fillcolor='rgba(0,100,80,0.2)',
+                line=dict(width=0)  # keep it as a filled area
             ), row=row, col=col)
-    
+
+            # weight label near the bottom
+            y_post = float(np.nanmin(freq_data[column])) * 0.95
+            x_offset = 0.5  # small shift within the 10-wide slot
+
             align_value = "left" if running_total == "Running total" else "center"
             valign_value = "bottom" if running_total == "Running total" else "middle"
-                                            
-            # Add annotation for weight at the bottom
+
             fig.add_annotation(
-                x=x_values[0] + x_offset,
+                x=x_vals[0] + x_offset,
                 y=y_post,
                 text=f"Weight: {row_data['weight']:.2f}",
                 showarrow=False,
-                font=dict(
-                    color="black",
-                    size=12),
+                font=dict(color="black", size=12),
                 align=align_value,
                 valign=valign_value,
                 textangle=270,
                 row=row, col=col
                 )
-            
-            # Add labels for values only in bursts
-            fig.add_trace(go.Scatter(
-            x=x_values, y=y_values, mode='lines+markers+text', name=column,
-            line_shape='linear',
-            hoverinfo='text',
-            hovertext=[f"Year: {index}<br>Frequency: {freq}" for index, freq in zip(freq_data.index, freq_data[column])],
-            text=y_values,
-            textposition='top center'
-        ), row=row, col=col)
-            print(freq_data[column])
-
 
         col += 1
         if col > 2:
             col = 1
             row += 1
-                
+
+    # Dynamic sizing
+    plot_height = num_rows * 500
+    plot_width = len(years) * spacing + padding
+
+    # Apply the same x settings to all subplots:
+    fig.update_xaxes(
+        range=[-spacing/2, x_positions[-1] + spacing/2],  # ±5 around the ends
+        tickmode='array',
+        tickvals=tickvals,
+        ticktext=ticktext
+    )
+
     fig.update_layout(
         showlegend=False,
         margin=dict(l=20, r=20, t=100, b=20),
-        height=num_rows * 500,
-        width=1500
+        height=plot_height,
+        width=plot_width,
+        autosize=False
     )
                 
-    fig.write_image("line_graph.png")
-    st.image("line_graph.png")
-    pio.write_image(fig, 'result.png', scale=4)
+    fig.write_image("line_graph.png", width=plot_width, height=plot_height)
+    
+    st.image("line_graph.png", use_column_width=False)
+    pio.write_image(fig, 'result.png', width=plot_width, height=plot_height, scale=4)
 
 @st.cache_data(ttl=3600)
 def download_result(freq_data, bursts):
@@ -496,7 +522,7 @@ if uploaded_file is not None:
                     st.info(f'We only detect a burst on {num_unique_labels} word(s), which is {top_n - num_unique_labels} fewer than the top word(s)', icon="ℹ️")
 
                 if viz_selected == "Line graph": 
-                    linegraph(bursts, freq_data)
+                    linegraph(bursts, freq_data, top_n)
                     
                 elif viz_selected =="Scatter plot":
                     scattervis(bursts, freq_data, top_n)
@@ -546,6 +572,7 @@ if uploaded_file is not None:
             st.button('👉 Press to download the list of detected bursts', on_click=None)
             st.text("Click download button.")
             
-    except:
+    except Exception as e:
+        st.write(e)
         st.error("Please ensure that your file or settings are correct. If you think there is a mistake, feel free to reach out to us!", icon="🚨")
         st.stop()

pages/6 Keywords Stem.py

@@ -291,10 +291,16 @@ if uploaded_file is not None:
             st.markdown('**Lamba, M., & Madhusudhan, M. (2021, July 31). Text Pre-Processing. Text Mining for Information Professionals, 79–103.** https://doi.org/10.1007/978-3-030-85085-2_3')
 
         with tab5:
-            st.text("Download keywords at bottom of table")
+            st.subheader(':blue[Result]', anchor=False)
+            st.button('Press to download result 👈')
+            st.text("Go to Result and click Download button.")  
+
             st.divider()
-            st.text("Download table")
-            st.markdown("![Downloading visualization](https://raw.githubusercontent.com/faizhalas/library-tools/mainimages/downloadtable.png")
+            st.subheader(':blue[List of Keywords]', anchor=False)
+            st.button('Press to download keywords 👈')
+            st.text("Go to List of Keywords and click Download button.") 
+            
     except:
         st.error("Please ensure that your file is correct. Please contact us if you find that this is an error.", icon="🚨")
         st.stop()     
+

pages/7 Sentiment Analysis.py

@@ -131,7 +131,7 @@ def conv_json(extype):
     keywords.rename(columns=col_dict,inplace=True)
     return keywords
 
-@st.cache_resource(ttl=3600)
+@st.cache_data(ttl=3600)
 def conv_pub(extype):
     if (get_ext(extype)).endswith('.tar.gz'):
         bytedata = extype.read()
@@ -168,8 +168,8 @@ if uploaded_file is not None:
             'Choose method',[
             'TextBlob','NLTKvader']
         )
-        words_to_remove = c1.text_input("Remove specific words. Separate words by semicolons (;)")        
-        wordcount = c2.number_input(label = "Words displayed", min_value = 1, step = 1, value=5)-1
+        words_to_remove = c1.text_input("Remove specific words. Separate words by semicolons (;)", on_change=reset_all)        
+        wordcount = c2.number_input(label = "Words displayed", min_value = 1, step = 1, value=5, on_change=reset_all)-1
         rem_copyright = c1.toggle('Remove copyright statement', value=True, on_change=reset_all)
         rem_punc = c2.toggle('Remove punctuation', value=True, on_change=reset_all)
 
@@ -191,17 +191,23 @@ if uploaded_file is not None:
             paper[ColCho] = paper['Abstract_pre'].apply(lambda x: ' '.join([word for word in x.split() if word not in (stop)]))
               
             words_rmv = [word.strip() for word in words_to_remove.split(";")]
-            remove_dict = {word: None for word in words_rmv}
-            
-            @st.cache_resource(ttl=3600)
+
+            @st.cache_data(ttl=3600)
             def remove_words(text):
-                 words = text.split()
-                 cleaned_words = [word for word in words if word not in remove_dict]
-                 return ' '.join(cleaned_words) 
+                if not isinstance(text, str):
+                    return text  # skip NaN or non-string
+                
+                # Regex pattern: remove exact whole words in words_rmv
+                pattern = r'\b(?:' + "|".join(map(re.escape, words_rmv)) + r')\b'
+                cleaned_text = re.sub(pattern, '', text)
+                
+                # Remove double spaces created after removal
+                return " ".join(cleaned_text.split())
             
             paper['Sentences__'] = paper['Abstract_pre'].map(remove_words)
-
+            
             return paper
+            
         paper=clean_csv(extype) 
     
         if method == 'NLTKvader':
@@ -266,11 +272,11 @@ if uploaded_file is not None:
 
                 return phrase, phrasepolar, phrasesubject
 
-            @st.cache_resource(ttl=3600)
+            @st.cache_data(ttl=3600)
             def mergelist(data):
                 return ' '.join(data)
 
-            @st.cache_resource(ttl=3600)
+            @st.cache_data(ttl=3600)
             def assignscore(data):
                 if data>0:
                     return "Positive"
@@ -357,3 +363,4 @@ if uploaded_file is not None:
     except:
         st.error("Please ensure that your file is correct. Please contact us if you find that this is an error.", icon="🚨")
         st.stop()
+

pages/8 Shifterator.py

@@ -399,8 +399,9 @@ if uploaded_file is not None:
                     'Choose column',
                     (df_col_sel), on_change=reset_all)
         
-                list_words = paper[column_selected].values.tolist()
-                list_unique = sorted(list(set(list_words)))
+                list_words = paper[column_selected].dropna()  # remove NaN
+                list_words = [w for w in list_words if str(w).strip() != ""]  # remove empty strings
+                list_unique = sorted(set(list_words))
                 
                 if column_selected is not None:
                     label1 = col2.selectbox(
@@ -513,13 +514,13 @@ if uploaded_file is not None:
 
         with tab4:
             st.subheader(':blue[Result]', anchor=False)
-            st.button('📥 Download Graph')
+            st.button('📥 Download Graph', on_click="ignore")
             st.text("Click Download Graph button.")  
 
             st.divider()
             st.subheader(':blue[Shifterator Dataframe]', anchor=False)
-            st.button('📥 Click to download result')
-            st.text("Click the Download button to get the CSV result.") 
+            st.button('📥 Press to download result', on_click="ignore")
+            st.text("Click the Download button to get the CSV result.")
 
     except:
         st.error("Please ensure that your file is correct. Please contact us if you find that this is an error.", icon="🚨")