add outline and BERTinference

.gitattributes

@@ -1,34 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

bert_model.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:dc61799e024f5d62f883b9e04886c749468378c19ed0185311a5ce4031ae8a5d
-size 409205202

classification.py

@@ -15,18 +15,17 @@ def classify_by_topic(articles, central_topics):
         tokenizer = AutoTokenizer.from_pretrained(
             "distilbert-base-multilingual-cased")
 
-        # 将一个句子转换为一个向量
         def sentence_to_vector(sentence, context):
-            # 分词并添加特殊标记
+
             sentence = context[0]+context[1]+sentence*4+context[2]+context[3]
             tokens = tokenizer.encode_plus(
                 sentence, add_special_tokens=True, return_tensors="pt")
-            # 获取每个词的隐藏状态向量
+
             outputs = model(**tokens)
             hidden_states = outputs.last_hidden_state
-            # 计算平均向量作为句子向量
+
             vector = np.squeeze(torch.mean(
-                hidden_states, dim=1).detach().numpy())  # a 1 x d tensor
+                hidden_states, dim=1).detach().numpy())
             return vector
 
         # 获取一个句子的上下文
@@ -51,26 +50,23 @@ def classify_by_topic(articles, central_topics):
                 nnext_sentence = sentences[index+2]
             return (pprev_sentence, prev_sentence, next_sentence, nnext_sentence)
 
-        # 将每个文章句子和每个中心句子转换为向量
         doc_vectors = [sentence_to_vector(sentence, get_context(
             articles, i)) for i, sentence in enumerate(articles)]
         topic_vectors = [sentence_to_vector(sentence, get_context(
             central_topics, i)) for i, sentence in enumerate(central_topics)]
-        # 计算每个文章句子和每个中心句子之间的余弦相似度矩阵
+        # 计算余弦相似度矩阵
         cos_sim_matrix = cosine_similarity(doc_vectors, topic_vectors)
 
-        # print(cos_sim_matrix)
         return cos_sim_matrix
 
-    # 按照相似度矩阵分类文章，返回一个列表
+    # 分类文章
     def group_by_topic(articles, central_topics, similarity_matrix):
         group = []
-        original_articles = articles.copy()  # 保存一份原始的文章列表
-        # 用原始的文章列表替换预处理后的文章列表
+        original_articles = articles.copy()
         for article, similarity in zip(original_articles, similarity_matrix):
-            max_similarity = max(similarity)  # 取最高的相似度值
-            max_index = similarity.tolist().index(max_similarity)  # 取最高相似度值对应的索引
-            # print(max_similarity,max_index )
+            max_similarity = max(similarity)
+            max_index = similarity.tolist().index(max_similarity)
+
             group.append((article, central_topics[max_index]))
 
         return group
@@ -79,5 +75,4 @@ def classify_by_topic(articles, central_topics):
     similarity_matrix = compute_similarity(articles, central_topics)
     groups = group_by_topic(articles, central_topics, similarity_matrix)
 
-    # 返回分类后的列表
     return groups

inference.py

@@ -0,0 +1,75 @@
+import os
+import numpy as np
+import transformers
+import torch
+import torch.nn as nn
+from torch import cuda
+from transformers import BertTokenizer
+
+
+def encoder(max_len,text):
+
+    tokenizer = BertTokenizer.from_pretrained("bert-base-chinese")
+    tokenizer = tokenizer(
+        text,
+        padding = True,
+        truncation = True,
+        max_length = max_len,
+        return_tensors='pt'
+        )
+    input_ids = tokenizer['input_ids']
+    token_type_ids = tokenizer['token_type_ids']
+    attention_mask = tokenizer['attention_mask']
+    return input_ids,token_type_ids,attention_mask
+
+
+def predict(model,device,text):
+    model.to(device)
+    model.eval()
+    with torch.no_grad():
+        input_ids,token_type_ids,attention_mask = encoder(512,text)
+        input_ids,token_type_ids,attention_mask=input_ids.to(device),token_type_ids.to(device),attention_mask.to(device)
+        out_put = model(input_ids,token_type_ids,attention_mask)
+        # pre_numpy = out_put.cpu().numpy().tolist()
+        probs = torch.nn.functional.softmax(out_put).detach().cpu().numpy().tolist()
+        # print(probs)
+        return probs[0][1]
+ 
+
+class BertClassificationModel(nn.Module):
+    def __init__(self):
+        super(BertClassificationModel, self).__init__()   
+        pretrained_weights="bert-base-chinese"
+        self.bert = transformers.BertModel.from_pretrained(pretrained_weights)
+        for param in self.bert.parameters():
+            param.requires_grad = True    
+        self.dense = nn.Linear(768, 3)
+        
+    def forward(self, input_ids,token_type_ids,attention_mask):
+        bert_output = self.bert(input_ids=input_ids,token_type_ids=token_type_ids, attention_mask=attention_mask)
+        bert_cls_hidden_state = bert_output[1]
+        linear_output = self.dense(bert_cls_hidden_state)
+        return  linear_output
+        
+def inference_matrix(topics):
+    device = torch.device('cuda' if cuda.is_available() else 'cpu')
+    load_path = "TSA/bert_model.pkl"   
+    model = torch.load(load_path,map_location=torch.device(device))
+    matrix = np.zeros([len(topics),len(topics)],dtype=float)
+    for i,i_text in enumerate(topics):
+        for j,j_text in enumerate(topics):
+            if(i == j):
+                matrix[i][j] = 1
+            else:
+                test = i_text+"  是否包含  "+j_text
+                outputs = predict(model,device,test)
+                # outputs = model(ids, mask,token_type_ids)
+                # print(outputs)
+                matrix[i][j] = outputs
+
+    return matrix
+if __name__ == "__main__":
+
+    print("yes")
+    topics = ['在本次报告中我将介绍分布式并行加速算法模型架构内存和计算优化以及集群架构等关键技术', '在现代机器学习任务中大模型训练已成为解决复杂问题的重要手段', '首先分布式并行加速策略包括数据并行模型并行流水线并行和张量并行等四种方式', '选择合适的集群架构是实现大模型的分布式训练的关键', '这些策略帮助我们将训练数据和模型分布到多个设备上以加速大模型训练过程']
+    print(inference_matrix(topics))

outline.py

@@ -0,0 +1,31 @@
+import numpy as np
+from scipy.cluster.hierarchy import linkage, fcluster, dendrogram
+import matplotlib.pyplot as plt
+
+def passage_outline(matrix,sentences):
+    # matrix = np.array([[1.0, 0.8, 0.2, 0.1],
+    #                 [0.8, 1.0, 0.3, 0.2],
+    #                 [0.2, 0.3, 1.0, 0.9],
+    #                 [0.1, 0.2, 0.9, 1.0]])
+    # sentences = ["主题句子1", "主题句子2", "主题句子3", "主题句子4"]
+
+    Z = linkage(matrix, method="average")
+
+    labels = fcluster(Z, t=0.5, criterion="distance")
+
+
+    # 根据簇标签和主题句子生成文章结构
+    structure = {}
+    for label, sentence in zip(labels, sentences):
+        if label not in structure:
+            structure[label] = []
+        structure[label].append(sentence)
+    outline = ""
+    outline_list = []
+    for key in sorted(structure.keys()):
+        outline_list.append(f"主题{key}：")
+        outline = outline+f"主题{key}：\n"
+        for sentence in structure[key]:
+            outline_list.append(sentence)
+            outline = outline+f"- {sentence}\n"
+    return outline,outline_list

run.py

@@ -1,7 +1,9 @@
 import util
-import TSA.abstruct as abstruct
+import abstract
 import classification
-
+import inference
+import outline
+from inference import BertClassificationModel
 # input:file/text,topic_num,max_length,output_choice
 # output:file/text/topic_sentence
 
@@ -29,15 +31,26 @@ article = util.seg(text)
 print(article)
 
 sentences = [util.clean_text(sentence) for sentence in article]
-central_sentences = abstruct.abstruct_main(sentences, topic_num)
+
+central_sentences = abstract.abstruct_main(sentences, topic_num)
 print(central_sentences)
+
 groups = classification.classify_by_topic(article, central_sentences)
 print(groups)
 
 groups = util.article_to_group(groups, central_sentences)
-ans = util.generation(groups, max_length)
+
+title_dict,title = util.generation(groups, max_length)
 # ans:
-# {(main_sentence,(Ai_abstruct,paragraph))}
-for i in ans.items():
+# {Ai_abstruct:(main_sentence,paragraph)}
+for i in title_dict.items():
     print(i)
-``
+
+matrix = inference.inference_matrix(title)
+print(matrix)
+
+text_outline,outline_list = outline.passage_outline(matrix,title)
+print(text_outline)
+
+output = util.formate_text(title_dict,outline_list)
+print (output)

util.py

@@ -3,10 +3,12 @@ import jieba
 import re
 import requests
 import backoff
+import time
 
 
 @backoff.on_exception(backoff.expo, requests.exceptions.RequestException)
 def post_url(url, headers, payload):
+    time.sleep(0.3)
     response = requests.request("POST", url, headers=headers, data=payload)
     return response
 
@@ -55,7 +57,7 @@ def generation(para, max_length):
 
     url = "https://aip.baidubce.com/rpc/2.0/nlp/v1/news_summary?charset=UTF-8&access_token=" + get_access_token()
     topic = {}
-
+    Ai_abstract = []
     for i, (j, k) in enumerate(para.items()):
         input_text = k
         # print(k)
@@ -71,5 +73,15 @@ def generation(para, max_length):
         response = post_url(url, headers, payload)
         text_dict = json.loads(response.text)
         # print(text_dict)
-        topic[j] = (text_dict['summary'], k)
-    return topic
+        topic[text_dict['summary']] = (j, k)
+        Ai_abstract.append(text_dict['summary'])
+    return topic,Ai_abstract
+def formate_text(title_dict,outline_list):
+    formated = []
+    for each in outline_list:
+        if(each not in title_dict.keys()):
+            formated.append(f"# {each}")
+        if(each in title_dict.keys()):
+            formated.append(f"## {each}")
+            formated.append(title_dict[each][1])
+    return formated