DevBM commited on
Commit
99220ed
1 Parent(s): 843d718

Update c1.py

Browse files
Files changed (1) hide show
  1. c1.py +72 -72
c1.py CHANGED
@@ -1,72 +1,72 @@
1
- import streamlit as st
2
- from transformers import T5ForConditionalGeneration, T5Tokenizer
3
- import torch
4
- import spacy
5
- import nltk
6
-
7
- #nltk.download('punkt')
8
- from nltk.tokenize import sent_tokenize
9
-
10
- # Load spaCy model
11
- nlp = spacy.load("en_core_web_sm")
12
-
13
- # Load T5 model and tokenizer
14
- model_name = "DevBM/t5-large-squad"
15
- model = T5ForConditionalGeneration.from_pretrained(model_name)
16
- tokenizer = T5Tokenizer.from_pretrained(model_name)
17
-
18
- # Function to extract keywords using spaCy
19
- def extract_keywords(text):
20
- doc = nlp(text)
21
- keywords = set()
22
- # Extract named entities
23
- for entity in doc.ents:
24
- keywords.add(entity.text)
25
- # Extract nouns and proper nouns
26
- for token in doc:
27
- if token.pos_ in ["NOUN", "PROPN"]:
28
- keywords.add(token.text)
29
- return list(keywords)
30
-
31
- # Function to map keywords to sentences
32
- def map_keywords_to_sentences(text, keywords):
33
- sentences = sent_tokenize(text)
34
- keyword_sentence_mapping = {}
35
- for keyword in keywords:
36
- for i, sentence in enumerate(sentences):
37
- if keyword in sentence:
38
- # Combine current sentence with surrounding sentences for context
39
- start = max(0, i-1)
40
- end = min(len(sentences), i+2)
41
- context = ' '.join(sentences[start:end])
42
- if keyword not in keyword_sentence_mapping:
43
- keyword_sentence_mapping[keyword] = context
44
- else:
45
- keyword_sentence_mapping[keyword] += ' ' + context
46
- return keyword_sentence_mapping
47
-
48
- # Function to generate questions
49
- def generate_question(context, answer):
50
- input_text = f"<context> {context} <answer> {answer}"
51
- input_ids = tokenizer.encode(input_text, return_tensors='pt')
52
- outputs = model.generate(input_ids)
53
- question = tokenizer.decode(outputs[0], skip_special_tokens=True)
54
- return question
55
-
56
- # Streamlit interface
57
- st.title("Question Generator from Text")
58
- text = st.text_area("Enter text here:")
59
- if st.button("Generate Questions"):
60
- if text:
61
- keywords = extract_keywords(text)
62
- keyword_sentence_mapping = map_keywords_to_sentences(text, keywords)
63
-
64
- st.subheader("Generated Questions:")
65
- for keyword, context in keyword_sentence_mapping.items():
66
- question = generate_question(context, keyword)
67
- st.write(f"**Context:** {context}")
68
- st.write(f"**Answer:** {keyword}")
69
- st.write(f"**Question:** {question}")
70
- st.write("---")
71
- else:
72
- st.write("Please enter some text to generate questions.")
 
1
+ import streamlit as st
2
+ from transformers import T5ForConditionalGeneration, T5Tokenizer
3
+ import torch
4
+ import spacy
5
+ import nltk
6
+ from b import b
7
+ #nltk.download('punkt')
8
+ from nltk.tokenize import sent_tokenize
9
+
10
+ # Load spaCy model
11
+ nlp = spacy.load("en_core_web_sm")
12
+
13
+ # Load T5 model and tokenizer
14
+ model_name = "DevBM/t5-large-squad"
15
+ model = T5ForConditionalGeneration.from_pretrained(model_name)
16
+ tokenizer = T5Tokenizer.from_pretrained(model_name)
17
+
18
+ # Function to extract keywords using spaCy
19
+ def extract_keywords(text):
20
+ doc = nlp(text)
21
+ keywords = set()
22
+ # Extract named entities
23
+ for entity in doc.ents:
24
+ keywords.add(entity.text)
25
+ # Extract nouns and proper nouns
26
+ for token in doc:
27
+ if token.pos_ in ["NOUN", "PROPN"]:
28
+ keywords.add(token.text)
29
+ return list(keywords)
30
+
31
+ # Function to map keywords to sentences
32
+ def map_keywords_to_sentences(text, keywords):
33
+ sentences = sent_tokenize(text)
34
+ keyword_sentence_mapping = {}
35
+ for keyword in keywords:
36
+ for i, sentence in enumerate(sentences):
37
+ if keyword in sentence:
38
+ # Combine current sentence with surrounding sentences for context
39
+ start = max(0, i-1)
40
+ end = min(len(sentences), i+2)
41
+ context = ' '.join(sentences[start:end])
42
+ if keyword not in keyword_sentence_mapping:
43
+ keyword_sentence_mapping[keyword] = context
44
+ else:
45
+ keyword_sentence_mapping[keyword] += ' ' + context
46
+ return keyword_sentence_mapping
47
+
48
+ # Function to generate questions
49
+ def generate_question(context, answer):
50
+ input_text = f"<context> {context} <answer> {answer}"
51
+ input_ids = tokenizer.encode(input_text, return_tensors='pt')
52
+ outputs = model.generate(input_ids)
53
+ question = tokenizer.decode(outputs[0], skip_special_tokens=True)
54
+ return question
55
+
56
+ # Streamlit interface
57
+ st.title("Question Generator from Text")
58
+ text = st.text_area("Enter text here:")
59
+ if st.button("Generate Questions"):
60
+ if text:
61
+ keywords = extract_keywords(text)
62
+ keyword_sentence_mapping = map_keywords_to_sentences(text, keywords)
63
+
64
+ st.subheader("Generated Questions:")
65
+ for keyword, context in keyword_sentence_mapping.items():
66
+ question = generate_question(context, keyword)
67
+ st.write(f"**Context:** {context}")
68
+ st.write(f"**Answer:** {keyword}")
69
+ st.write(f"**Question:** {question}")
70
+ st.write("---")
71
+ else:
72
+ st.write("Please enter some text to generate questions.")