1

Biweekly Feedback Summary.md

@@ -0,0 +1,19 @@
+# Biweekly Feedback Summary
+
+## 21.10.2022
+1. [new dataset](https://huggingface.co/datasets/Adapting/abstract-keyphrases)
+1. new model architecture:[KeyBartAdapter](https://github.com/Mondkuchen/idp_LiteratureResearch_Tool/blob/main/lrt/clustering/models/keyBartPlus.py)
+    - [train script](https://github.com/Mondkuchen/idp_LiteratureResearch_Tool/blob/main/scripts/train/train.py)
+    - [training result](https://huggingface.co/Adapting/KeyBartAdapter)
+2. [文献分析平台比较](https://leoxiang66.github.io/LRT-Doc/4-%E6%96%87%E7%8C%AE%E5%88%86%E6%9E%90%E5%B9%B3%E5%8F%B0%E6%AF%94%E8%BE%83/)
+
+## TODOs
+- [x] a new keywords generation model
+- [x] 比较其他文献工具
+- [x] 开始研究query API
+- [ ] 可视化
+- [ ] 和其他文献工具比较结果
+- [ ] 收集更多训练数据/fine-tune
+- [ ] add clustering: using other clustering algorithms such as Gausian Mixture Model (GMM)
+- [ ] add dimension reduction
+- [ ] [better PLM](https://huggingface.co/spaces/mteb/leaderboard)

scripts/inference/inference.py

@@ -0,0 +1,17 @@
+if __name__ == '__main__':
+    import sys
+    from pathlib import Path
+
+    project_root = Path(__file__).parent.parent.parent.absolute()  # /home/adapting/git/leoxiang66/idp_LiteratureResearch_Tool
+    sys.path.append(project_root.__str__())
+
+    from transformers import Text2TextGenerationPipeline
+    from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
+    tokenizer = AutoTokenizer.from_pretrained("Adapting/KeyBartAdapter")
+    model = AutoModelForSeq2SeqLM.from_pretrained("Adapting/KeyBartAdapter")
+
+    pipe = Text2TextGenerationPipeline(model=model, tokenizer=tokenizer)
+
+    abstract = '''Non-referential face image quality assessment methods have gained popularity as a pre-filtering step on face recognition systems. In most of them, the quality score is usually designed with face matching in mind. However, a small amount of work has been done on measuring their impact and usefulness on Presentation Attack Detection (PAD). In this paper, we study the effect of quality assessment methods on filtering bona fide and attack samples, their impact on PAD systems, and how the performance of such systems is improved when training on a filtered (by quality) dataset. On a Vision Transformer PAD algorithm, a reduction of 20% of the training dataset by removing lower quality samples allowed us to improve the BPCER by 3% in a cross-dataset test.'''
+
+    print(pipe(abstract))

scripts/inference/lrt.ipynb

@@ -0,0 +1,310 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "collapsed_sections": [],
+      "machine_shape": "hm",
+      "mount_file_id": "1aBrZOQRBhTOgg2wvc0sh1d79m9abNU-O",
+      "authorship_tag": "ABX9TyOdcckjc7kMuJJm+A64/dzt",
+      "include_colab_link": true
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    },
+    "accelerator": "GPU"
+  },
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "view-in-github",
+        "colab_type": "text"
+      },
+      "source": [
+        "<a href=\"https://colab.research.google.com/github/Mondkuchen/idp_LiteratureResearch_Tool/blob/main/scripts/inference/lrt.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [],
+      "metadata": {
+        "id": "NDK6pgcVQ6RI"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from google.colab import drive\n",
+        "drive.mount('/content/drive')"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "L76IjCQkviFl",
+        "outputId": "eebb493e-ff37-4336-9a03-8b39307627fd"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Mounted at /content/drive\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "%cd /content/drive/MyDrive/git/idp_LiteratureResearch_Tool/\n"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "PnedAltsxot6",
+        "outputId": "0de30b5e-0ce2-4adf-aff0-7e952e5087c3"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "/content/drive/MyDrive/git/idp_LiteratureResearch_Tool\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "!ls"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "CPRrgG9Fx06U",
+        "outputId": "62224f1a-a049-4c40-89a5-4f4a1b888842"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "example_run.py\tliterature  README.md  requirements.txt\n",
+            "examples\tlrt\t    reports    setup.py\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "!pip install -r requirements.txt"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 1000
+        },
+        "id": "w2ruvvI-yLeD",
+        "outputId": "58b61e2e-42a0-462b-8745-934b14aee1fd"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
+            "Collecting evaluate==0.2.2\n",
+            "  Downloading evaluate-0.2.2-py3-none-any.whl (69 kB)\n",
+            "\u001b[K     |████████████████████████████████| 69 kB 4.9 MB/s \n",
+            "\u001b[?25hCollecting kmeans_pytorch==0.3\n",
+            "  Downloading kmeans_pytorch-0.3-py3-none-any.whl (4.4 kB)\n",
+            "Requirement already satisfied: numpy==1.21.6 in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 3)) (1.21.6)\n",
+            "Requirement already satisfied: scikit_learn==1.0.2 in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 4)) (1.0.2)\n",
+            "Collecting sentence_transformers==2.2.2\n",
+            "  Downloading sentence-transformers-2.2.2.tar.gz (85 kB)\n",
+            "\u001b[K     |████████████████████████████████| 85 kB 4.9 MB/s \n",
+            "\u001b[?25hCollecting setuptools==63.4.1\n",
+            "  Downloading setuptools-63.4.1-py3-none-any.whl (1.2 MB)\n",
+            "\u001b[K     |████████████████████████████████| 1.2 MB 47.7 MB/s \n",
+            "\u001b[?25hRequirement already satisfied: torch==1.12.1 in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 7)) (1.12.1+cu113)\n",
+            "Requirement already satisfied: yellowbrick==1.5 in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 8)) (1.5)\n",
+            "Collecting transformers==4.22.1\n",
+            "  Downloading transformers-4.22.1-py3-none-any.whl (4.9 MB)\n",
+            "\u001b[K     |████████████████████████████████| 4.9 MB 56.6 MB/s \n",
+            "\u001b[?25hCollecting textdistance==4.5.0\n",
+            "  Downloading textdistance-4.5.0-py3-none-any.whl (31 kB)\n",
+            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from evaluate==0.2.2->-r requirements.txt (line 1)) (2.23.0)\n",
+            "Requirement already satisfied: fsspec[http]>=2021.05.0 in /usr/local/lib/python3.7/dist-packages (from evaluate==0.2.2->-r requirements.txt (line 1)) (2022.8.2)\n",
+            "Collecting multiprocess\n",
+            "  Downloading multiprocess-0.70.13-py37-none-any.whl (115 kB)\n",
+            "\u001b[K     |████████████████████████████████| 115 kB 65.0 MB/s \n",
+            "\u001b[?25hRequirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.7/dist-packages (from evaluate==0.2.2->-r requirements.txt (line 1)) (4.64.1)\n",
+            "Collecting responses<0.19\n",
+            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
+            "Collecting huggingface-hub>=0.7.0\n",
+            "  Downloading huggingface_hub-0.10.0-py3-none-any.whl (163 kB)\n",
+            "\u001b[K     |████████████████████████████████| 163 kB 60.3 MB/s \n",
+            "\u001b[?25hRequirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from evaluate==0.2.2->-r requirements.txt (line 1)) (0.3.5.1)\n",
+            "Collecting datasets>=2.0.0\n",
+            "  Downloading datasets-2.5.1-py3-none-any.whl (431 kB)\n",
+            "\u001b[K     |████████████████████████████████| 431 kB 51.2 MB/s \n",
+            "\u001b[?25hCollecting xxhash\n",
+            "  Downloading xxhash-3.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
+            "\u001b[K     |████████████████████████████████| 212 kB 52.0 MB/s \n",
+            "\u001b[?25hRequirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from evaluate==0.2.2->-r requirements.txt (line 1)) (4.12.0)\n",
+            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from evaluate==0.2.2->-r requirements.txt (line 1)) (1.3.5)\n",
+            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from evaluate==0.2.2->-r requirements.txt (line 1)) (21.3)\n",
+            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit_learn==1.0.2->-r requirements.txt (line 4)) (3.1.0)\n",
+            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit_learn==1.0.2->-r requirements.txt (line 4)) (1.1.0)\n",
+            "Requirement already satisfied: scipy>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from scikit_learn==1.0.2->-r requirements.txt (line 4)) (1.7.3)\n",
+            "Requirement already satisfied: torchvision in /usr/local/lib/python3.7/dist-packages (from sentence_transformers==2.2.2->-r requirements.txt (line 5)) (0.13.1+cu113)\n",
+            "Requirement already satisfied: nltk in /usr/local/lib/python3.7/dist-packages (from sentence_transformers==2.2.2->-r requirements.txt (line 5)) (3.7)\n",
+            "Collecting sentencepiece\n",
+            "  Downloading sentencepiece-0.1.97-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
+            "\u001b[K     |████████████████████████████████| 1.3 MB 53.7 MB/s \n",
+            "\u001b[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch==1.12.1->-r requirements.txt (line 7)) (4.1.1)\n",
+            "Requirement already satisfied: matplotlib!=3.0.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from yellowbrick==1.5->-r requirements.txt (line 8)) (3.2.2)\n",
+            "Requirement already satisfied: cycler>=0.10.0 in /usr/local/lib/python3.7/dist-packages (from yellowbrick==1.5->-r requirements.txt (line 8)) (0.11.0)\n",
+            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers==4.22.1->-r requirements.txt (line 9)) (3.8.0)\n",
+            "Collecting tokenizers!=0.11.3,<0.13,>=0.11.1\n",
+            "  Downloading tokenizers-0.12.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.6 MB)\n",
+            "\u001b[K     |████████████████████████████████| 6.6 MB 40.9 MB/s \n",
+            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.22.1->-r requirements.txt (line 9)) (2022.6.2)\n",
+            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers==4.22.1->-r requirements.txt (line 9)) (6.0)\n",
+            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.7/dist-packages (from datasets>=2.0.0->evaluate==0.2.2->-r requirements.txt (line 1)) (3.8.1)\n",
+            "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets>=2.0.0->evaluate==0.2.2->-r requirements.txt (line 1)) (6.0.1)\n",
+            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets>=2.0.0->evaluate==0.2.2->-r requirements.txt (line 1)) (1.3.1)\n",
+            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets>=2.0.0->evaluate==0.2.2->-r requirements.txt (line 1)) (6.0.2)\n",
+            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets>=2.0.0->evaluate==0.2.2->-r requirements.txt (line 1)) (22.1.0)\n",
+            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets>=2.0.0->evaluate==0.2.2->-r requirements.txt (line 1)) (1.8.1)\n",
+            "Requirement already satisfied: asynctest==0.13.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets>=2.0.0->evaluate==0.2.2->-r requirements.txt (line 1)) (0.13.0)\n",
+            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets>=2.0.0->evaluate==0.2.2->-r requirements.txt (line 1)) (2.1.1)\n",
+            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets>=2.0.0->evaluate==0.2.2->-r requirements.txt (line 1)) (1.2.0)\n",
+            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets>=2.0.0->evaluate==0.2.2->-r requirements.txt (line 1)) (4.0.2)\n",
+            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib!=3.0.0,>=2.0.2->yellowbrick==1.5->-r requirements.txt (line 8)) (1.4.4)\n",
+            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib!=3.0.0,>=2.0.2->yellowbrick==1.5->-r requirements.txt (line 8)) (3.0.9)\n",
+            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib!=3.0.0,>=2.0.2->yellowbrick==1.5->-r requirements.txt (line 8)) (2.8.2)\n",
+            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.1->matplotlib!=3.0.0,>=2.0.2->yellowbrick==1.5->-r requirements.txt (line 8)) (1.15.0)\n",
+            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->evaluate==0.2.2->-r requirements.txt (line 1)) (2022.6.15)\n",
+            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->evaluate==0.2.2->-r requirements.txt (line 1)) (3.0.4)\n",
+            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->evaluate==0.2.2->-r requirements.txt (line 1)) (1.24.3)\n",
+            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->evaluate==0.2.2->-r requirements.txt (line 1)) (2.10)\n",
+            "Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1\n",
+            "  Downloading urllib3-1.25.11-py2.py3-none-any.whl (127 kB)\n",
+            "\u001b[K     |████████████████████████████████| 127 kB 53.0 MB/s \n",
+            "\u001b[?25hRequirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->evaluate==0.2.2->-r requirements.txt (line 1)) (3.8.1)\n",
+            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from nltk->sentence_transformers==2.2.2->-r requirements.txt (line 5)) (7.1.2)\n",
+            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->evaluate==0.2.2->-r requirements.txt (line 1)) (2022.2.1)\n",
+            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.7/dist-packages (from torchvision->sentence_transformers==2.2.2->-r requirements.txt (line 5)) (7.1.2)\n",
+            "Building wheels for collected packages: sentence-transformers\n",
+            "  Building wheel for sentence-transformers (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
+            "  Created wheel for sentence-transformers: filename=sentence_transformers-2.2.2-py3-none-any.whl size=125938 sha256=422c6b8ba07037cbc0021b7dd77779f2f4cabd92e9a6edd18099753cd88d92d1\n",
+            "  Stored in directory: /root/.cache/pip/wheels/bf/06/fb/d59c1e5bd1dac7f6cf61ec0036cc3a10ab8fecaa6b2c3d3ee9\n",
+            "Successfully built sentence-transformers\n",
+            "Installing collected packages: urllib3, xxhash, tokenizers, responses, multiprocess, huggingface-hub, transformers, sentencepiece, datasets, textdistance, setuptools, sentence-transformers, kmeans-pytorch, evaluate\n",
+            "  Attempting uninstall: urllib3\n",
+            "    Found existing installation: urllib3 1.24.3\n",
+            "    Uninstalling urllib3-1.24.3:\n",
+            "      Successfully uninstalled urllib3-1.24.3\n",
+            "  Attempting uninstall: setuptools\n",
+            "    Found existing installation: setuptools 57.4.0\n",
+            "    Uninstalling setuptools-57.4.0:\n",
+            "      Successfully uninstalled setuptools-57.4.0\n",
+            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
+            "ipython 7.9.0 requires jedi>=0.10, which is not installed.\n",
+            "numba 0.56.2 requires setuptools<60, but you have setuptools 63.4.1 which is incompatible.\u001b[0m\n",
+            "Successfully installed datasets-2.5.1 evaluate-0.2.2 huggingface-hub-0.10.0 kmeans-pytorch-0.3 multiprocess-0.70.13 responses-0.18.0 sentence-transformers-2.2.2 sentencepiece-0.1.97 setuptools-63.4.1 textdistance-4.5.0 tokenizers-0.12.1 transformers-4.22.1 urllib3-1.25.11 xxhash-3.0.0\n"
+          ]
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "application/vnd.colab-display-data+json": {
+              "pip_warning": {
+                "packages": [
+                  "pkg_resources"
+                ]
+              }
+            }
+          },
+          "metadata": {}
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "!python example_run.py"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "r5s28dVs4vmi",
+        "outputId": "17395da1-2d67-48ad-a4f4-e885dfedee77"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "metadata": {
+            "tags": null
+          },
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "The cache for model files in Transformers v4.22.0 has been updated. Migrating your old cache. This is a one-time only operation. You can interrupt this and resume the migration later on by calling `transformers.utils.move_cache()`.\n",
+            "Moving 0 files to the new cache system\n",
+            "0it [00:00, ?it/s]\n",
+            "Downloading: 100% 1.18k/1.18k [00:00<00:00, 1.23MB/s]\n",
+            "Downloading: 100% 190/190 [00:00<00:00, 183kB/s]\n",
+            "Downloading: 100% 10.6k/10.6k [00:00<00:00, 5.27MB/s]\n",
+            "Downloading: 100% 612/612 [00:00<00:00, 537kB/s]\n",
+            "Downloading: 100% 116/116 [00:00<00:00, 108kB/s]\n",
+            "Downloading: 100% 39.3k/39.3k [00:00<00:00, 628kB/s]\n",
+            "Downloading: 100% 90.9M/90.9M [00:01<00:00, 47.6MB/s]\n",
+            "Downloading: 100% 53.0/53.0 [00:00<00:00, 52.2kB/s]\n",
+            "Downloading: 100% 112/112 [00:00<00:00, 93.7kB/s]\n",
+            "Downloading: 100% 466k/466k [00:00<00:00, 1.49MB/s]\n",
+            "Downloading: 100% 350/350 [00:00<00:00, 299kB/s]\n",
+            "Downloading: 100% 13.2k/13.2k [00:00<00:00, 8.80MB/s]\n",
+            "Downloading: 100% 232k/232k [00:00<00:00, 1.24MB/s]\n",
+            "Downloading: 100% 349/349 [00:00<00:00, 293kB/s]\n",
+            "Downloading: 100% 1.92k/1.92k [00:00<00:00, 1.72MB/s]\n",
+            "Downloading: 100% 792k/792k [00:00<00:00, 12.8MB/s]\n",
+            "Downloading: 100% 2.42M/2.42M [00:00<00:00, 5.44MB/s]\n",
+            "Downloading: 100% 1.79k/1.79k [00:00<00:00, 1.58MB/s]\n",
+            "Downloading: 100% 1.38k/1.38k [00:00<00:00, 1.00MB/s]\n",
+            "Downloading: 100% 892M/892M [00:17<00:00, 51.9MB/s]\n",
+            ">>> pipeline starts...\n",
+            ">>> start generating word embeddings...\n",
+            ">>> successfully generated word embeddings...\n",
+            ">>> start clustering...\n",
+            ">>> The best K is 2.\n",
+            ">>> finished clustering...\n",
+            ">>> start keywords extraction\n",
+            ">>> finished keywords extraction\n",
+            ">>> pipeline finished!\n",
+            "\n",
+            "['machine translation/similar language translation/news translation', 'natural language processing/nlp/natural language inference', 'model pretraining/pretraining/pre-training', 'wmt 2020', 'model architecture']\n",
+            "['deep learning/bayesian deep learning/machine learning', 'scene reconstruction/face recognition', 'convolutional networks', 'ilsvr', 'classification']\n"
+          ]
+        }
+      ]
+    }
+  ]
+}

scripts/readme.md

@@ -0,0 +1,5 @@
+# Scripts
+This folder contains scripts for 
+- model training and evaluation
+- model inference
+- tests and debugging

scripts/tests/lrt_test_run.py

@@ -0,0 +1,65 @@
+if __name__ == '__main__':
+    import sys
+    from pathlib import Path
+
+    project_root = Path(
+        __file__).parent.parent.parent.absolute()  # /home/adapting/git/leoxiang66/idp_LiteratureResearch_Tool
+    sys.path.append(project_root.__str__())
+
+    from lrt import LiteratureResearchTool, Configuration
+    from lrt.utils import ArticleList
+    config = Configuration(
+        plm= 'all-mpnet-base-v2',
+        dimension_reduction='none',
+        clustering='kmeans-euclidean',
+        # keywords_extraction='KeyBartAdapter'
+        keywords_extraction= 'keyphrase-transformer'
+    )
+    # import evaluate
+    # import numpy as np
+
+
+    # accuracy_metric = evaluate.load("accuracy")
+    # # minimal_config = BaselineConfig
+
+    # sentences = [
+    #     "This paper presents the results of the news translation task and the similar language translation task, both organised alongside the Conference on Machine Translation (WMT) 2020. In the news task, participants were asked to build machine translation systems for any of 11 language pairs, to be evaluated on test sets consisting mainly of news stories. The task was also opened up to additional test suites to probe specific aspects of translation. In the similar language translation task, participants built machine translation systems for translating between closely related pairs of languages.",
+    #     "Recent progress in natural language processing has been driven by advances in both model architecture and model pretraining. Transformer architectures have facilitated building higher-capacity models and pretraining has made it possible to effectively utilize this capacity for a wide variety of tasks. Transformers is an open-source library with the goal of opening up these advances to the wider machine learning community. The library consists of carefully engineered state-of-the art Transformer architectures under a unified API. Backing this library is a curated collection of pretrained models made by and available for the community. Transformers is designed to be extensible by researchers, simple for practitioners, and fast and robust in industrial deployments. The library is available at https://github.com/huggingface/transformers.",
+    #     'Convolutional networks are at the core of most state of-the-art computer vision solutions for a wide variety of tasks. Since 2014 very deep convolutional networks started to become mainstream, yielding substantial gains in various benchmarks. Although increased model size and computational cost tend to translate to immediate quality gains for most tasks (as long as enough labeled data is provided for training), computational efficiency and low parameter count are still enabling factors for various use cases such as mobile vision and big-data scenarios. Here we are exploring ways to scale up networks in ways that aim at utilizing the added computation as efficiently as possible by suitably factorized convolutions and aggressive regularization. We benchmark our methods on the ILSVRC 2012 classification challenge validation set demonstrate substantial gains over the state of the art: 21:2% top-1 and 5:6% top-5 error for single frame evaluation using a network with a computational cost of 5 billion multiply-adds per inference and with using less than 25 million parameters. With an ensemble of 4 models and multi-crop evaluation, we report 3:5% top-5 error and 17:3% top-1 error on the validation set and 3:6% top-5 error on the official test set.',
+    #     'Deep learning is at the heart of the current rise of artificial intelligence. In the field of computer vision, it has become the workhorse for applications ranging from self-driving cars to surveillance and security. Whereas, deep neural networks have demonstrated phenomenal success (often beyond human capabilities) in solving complex problems, recent studies show that they are vulnerable to adversarial attacks in the form of subtle perturbations to inputs that lead a model to predict incorrect outputs. For images, such perturbations are often too small to be perceptible, yet they completely fool the deep learning models. Adversarial attacks pose a serious threat to the success of deep learning in practice. This fact has recently led to a large influx of contributions in this direction. This paper presents the first comprehensive survey on adversarial attacks on deep learning in computer vision. We review the works that design adversarial attacks, analyze the existence of such attacks and propose defenses against them. To emphasize that adversarial attacks are possible in practical conditions, we separately review the contributions that evaluate adversarial attacks in the real-world scenarios. Finally, drawing on the reviewed literature, we provide a broader outlook of this research direction.',
+    #     '''Feed-forward layers constitute two-thirds of a transformer model's parameters, yet their role in the network remains under-explored. We show that feed-forward layers in transformer-based language models operate as key-value memories, where each key correlates with textual patterns in the training examples, and each value induces a distribution over the output vocabulary. Our experiments show that the learned patterns are human-interpretable, and that lower layers tend to capture shallow patterns, while upper layers learn more semantic ones. The values complement the keys' input patterns by inducing output distributions that concentrate probability mass on tokens likely to appear immediately after each pattern, particularly in the upper layers. Finally, we demonstrate that the output of a feed-forward layer is a composition of its memories, which is subsequently refined throughout the model's layers via residual connections to produce the final output distribution.''',
+    #     '''Bidirectional Encoder Representations from Transformers (BERT) has shown marvelous improvements across various NLP tasks, and consecutive variants have been proposed to further improve the performance of the pre-trained language models. In this paper, we target on revisiting Chinese pre-trained language models to examine their effectiveness in a non-English language and release the Chinese pre-trained language model series to the community. We also propose a simple but effective model called MacBERT, which improves upon RoBERTa in several ways, especially the masking strategy that adopts MLM as correction (Mac). We carried out extensive experiments on eight Chinese NLP tasks to revisit the existing pre-trained language models as well as the proposed MacBERT. Experimental results show that MacBERT could achieve state-of-the-art performances on many NLP tasks, and we also ablate details with several findings that may help future research. https://github.com/ymcui/MacBERT''',
+    #     '''From the Publisher: A basic problem in computer vision is to understand the structure of a real world scene given several images of it. Recent major developments in the theory and practice of scene reconstruction are described in detail in a unified framework. The book covers the geometric principles and how to represent objects algebraically so they can be computed and applied. The authors provide comprehensive background material and explain how to apply the methods and implement the algorithms directly.''',
+    #     '''There are two major types of uncertainty one can model. Aleatoric uncertainty captures noise inherent in the observations. On the other hand, epistemic uncertainty accounts for uncertainty in the model -- uncertainty which can be explained away given enough data. Traditionally it has been difficult to model epistemic uncertainty in computer vision, but with new Bayesian deep learning tools this is now possible. We study the benefits of modeling epistemic vs. aleatoric uncertainty in Bayesian deep learning models for vision tasks. For this we present a Bayesian deep learning framework combining input-dependent aleatoric uncertainty together with epistemic uncertainty. We study models under the framework with per-pixel semantic segmentation and depth regression tasks. Further, our explicit uncertainty formulation leads to new loss functions for these tasks, which can be interpreted as learned attenuation. This makes the loss more robust to noisy data, also giving new state-of-the-art results on segmentation and depth regression benchmarks.''',
+    #     '''Language model pre-training, such as BERT, has significantly improved the performances of many natural language processing tasks. However, pre-trained language models are usually computationally expensive, so it is difficult to efficiently execute them on resource-restricted devices. To accelerate inference and reduce model size while maintaining accuracy, we first propose a novel Transformer distillation method that is specially designed for knowledge distillation (KD) of the Transformer-based models. By leveraging this new KD method, the plenty of knowledge encoded in a large “teacher” BERT can be effectively transferred to a small “student” TinyBERT. Then, we introduce a new two-stage learning framework for TinyBERT, which performs Transformer distillation at both the pre-training and task-specific learning stages. This framework ensures that TinyBERT can capture the general-domain as well as the task-specific knowledge in BERT. TinyBERT4 with 4 layers is empirically effective and achieves more than 96.8% the performance of its teacher BERT-Base on GLUE benchmark, while being 7.5x smaller and 9.4x faster on inference. TinyBERT4 is also significantly better than 4-layer state-of-the-art baselines on BERT distillation, with only ~28% parameters and ~31% inference time of them. Moreover, TinyBERT6 with 6 layers performs on-par with its teacher BERT-Base.''',
+    #     '''This paper presents SimCSE, a simple contrastive learning framework that greatly advances the state-of-the-art sentence embeddings. We first describe an unsupervised approach, which takes an input sentence and predicts itself in a contrastive objective, with only standard dropout used as noise. This simple method works surprisingly well, performing on par with previous supervised counterparts. We hypothesize that dropout acts as minimal data augmentation and removing it leads to a representation collapse. Then, we draw inspiration from the recent success of learning sentence embeddings from natural language inference (NLI) datasets and incorporate annotated pairs from NLI datasets into contrastive learning by using entailment pairs as positives and contradiction pairs as hard negatives. We evaluate SimCSE on standard semantic textual similarity (STS) tasks, and our unsupervised and supervised models using BERT-base achieve an average of 74.5% and 81.6% Spearman's correlation respectively, a 7.9 and 4.6 points improvement compared to previous best results. We also show that contrastive learning theoretically regularizes pre-trained embeddings' anisotropic space to be more uniform, and it better aligns positive pairs when supervised signals are available.''',
+    #     '''Over the last years deep learning methods have been shown to outperform previous state-of-the-art machine learning techniques in several fields, with computer vision being one of the most prominent cases. This review paper provides a brief overview of some of the most significant deep learning schemes used in computer vision problems, that is, Convolutional Neural Networks, Deep Boltzmann Machines and Deep Belief Networks, and Stacked Denoising Autoencoders. A brief account of their history, structure, advantages, and limitations is given, followed by a description of their applications in various computer vision tasks, such as object detection, face recognition, action and activity recognition, and human pose estimation. Finally, a brief overview is given of future directions in designing deep learning schemes for computer vision problems and the challenges involved therein.''',
+    #     '''Computer vision is an interdisciplinary scientific field that deals with how computers can gain high-level understanding from digital images or videos. From the perspective of engineering, it seeks to understand and automate tasks that the human visual system can do. Computer vision tasks include methods for acquiring, processing, analyzing and understanding digital images, and extraction of high-dimensional data from the real world in order to produce numerical or symbolic information, e.g. in the forms of decisions.[3][4][5][6] Understanding in this context means the transformation of visual images (the input of the retina) into descriptions of the world that make sense to thought processes and can elicit appropriate action. This image understanding can be seen as the disentangling of symbolic information from image data using models constructed with the aid of geometry, physics, statistics, and learning theory.''',
+    #
+    # ]
+
+
+
+    lrt = LiteratureResearchTool(config)
+    platforms = [
+        'IEEE',
+        # 'Arxiv',
+        # 'Paper with Code'
+                 ]
+    ret = lrt('machine learning',100,2020,2022,platforms, best_k=5)
+    for plat in platforms:
+        clusters, articles = next(ret)
+        print(plat)
+        print(clusters)
+        print('keyphrases:')
+        for c in clusters:
+            print(c.top_5_keyphrases)
+
+            # 打印每个cluster包含的articles
+            # ids = c.elements()
+            # articles_in_cluster = ArticleList([articles[i] for i in ids] )
+            # print(articles_in_cluster)
+        print()
+
+
+

scripts/tests/model_test.py

@@ -0,0 +1,103 @@
+if __name__ == '__main__':
+    import sys
+    from pathlib import Path
+
+    project_root = Path(
+        __file__).parent.parent.parent.absolute()  # /home/adapting/git/leoxiang66/idp_LiteratureResearch_Tool
+    sys.path.append(project_root.__str__())
+
+    import torch
+    from lrt.clustering.models.keyBartPlus import *
+    from lrt.clustering.models.adapter import *
+    from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
+    import os
+
+    ####################### Adapter Test #############################
+    input_dim = 1024
+    adapter_hid_dim = 256
+    adapter = Adapter(input_dim,adapter_hid_dim)
+
+    data = torch.randn(10, 20, input_dim)
+
+    tmp = adapter(data)
+
+    assert data.size() == tmp.size()
+    ####################### Adapter Test #############################
+
+    ####################### BartDecoderPlus Test #############################
+    keyBart = AutoModelForSeq2SeqLM.from_pretrained("bloomberg/KeyBART")
+    bartDecoderP = BartDecoderPlus(keyBart, 100)
+    tmp = bartDecoderP(inputs_embeds=data,
+                       output_attentions = True,
+                       output_hidden_states = True,
+                       encoder_hidden_states = data
+                       )
+    print(type(tmp))
+    # print(tmp.__dict__)
+    print(dir(tmp))
+    last_hid_states = tmp.last_hidden_state
+    hidden_states = tmp.hidden_states
+    attentions = tmp.attentions
+    cross_attention = tmp.cross_attentions
+    print(last_hid_states.shape)
+    print(hidden_states.__len__())
+    print(attentions.__len__())
+    print(len(cross_attention))
+    # print(cross_attention[0])
+    print(cross_attention[0].shape)
+
+    ####################### BartDecoderPlus Test #############################
+
+    ####################### BartPlus Test #############################
+    bartP = BartPlus(keyBart,100)
+    tmp = bartP(
+        inputs_embeds = data,
+        decoder_inputs_embeds = data,
+        output_attentions=True,
+        output_hidden_states=True,
+                )
+    print(type(tmp))
+    # print(tmp.__dict__)
+    print(dir(tmp))
+    last_hid_states = tmp.last_hidden_state
+    hidden_states = tmp.decoder_hidden_states
+    attentions = tmp.decoder_attentions
+    cross_attention = tmp.cross_attentions
+    print(last_hid_states.shape)
+    print(hidden_states.__len__())
+    print(attentions.__len__())
+    print(len(cross_attention))
+    # print(cross_attention[0])
+    print(cross_attention[0].shape)
+    ####################### BartPlus Test #############################
+
+    ####################### Summary #############################
+    from torchinfo import summary
+
+    summary(bartP)
+    # summary(bartDecoderP)
+    ####################### Summary #############################
+
+    ####################### KeyBartAdapter Test #############################
+    keybart_adapter = KeyBartAdapter(100)
+    tmp = keybart_adapter(
+        inputs_embeds=data,
+        decoder_inputs_embeds=data,
+        output_attentions=True,
+        output_hidden_states=True,
+    )
+    print(type(tmp))
+    # print(tmp.__dict__)
+    print(dir(tmp))
+    last_hid_states = tmp.encoder_last_hidden_state
+    hidden_states = tmp.decoder_hidden_states
+    attentions = tmp.decoder_attentions
+    cross_attention = tmp.cross_attentions
+    print(last_hid_states.shape)
+    print(hidden_states.__len__())
+    print(attentions.__len__())
+    print(len(cross_attention))
+    # print(cross_attention[0])
+    print(cross_attention[0].shape)
+    summary(keybart_adapter)
+    ####################### KeyBartAdapter Test #############################

scripts/train/train.py

@@ -0,0 +1,171 @@
+def train(
+        push_to_hub:bool,
+        num_epoch: int,
+        train_batch_size: int,
+        eval_batch_size: int,
+):
+    import torch
+    import numpy as np
+
+    # 1. Dataset
+    from datasets import load_dataset
+    dataset = load_dataset("Adapting/abstract-keyphrases")
+
+    # 2. Model
+    from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
+    from lrt.clustering.models import KeyBartAdapter
+    tokenizer = AutoTokenizer.from_pretrained("Adapting/KeyBartAdapter")
+
+    '''
+    Or you can just use the initial model weights from Huggingface:
+    model = AutoModelForSeq2SeqLM.from_pretrained("Adapting/KeyBartAdapter",
+                                                  revision='9c3ed39c6ed5c7e141363e892d77cf8f589d5999')
+    '''
+
+    model = KeyBartAdapter(256)
+
+    # 3. preprocess dataset
+    dataset = dataset.shuffle()
+
+    def preprocess_function(examples):
+        inputs = examples['Abstract']
+        targets = examples['Keywords']
+        model_inputs = tokenizer(inputs, truncation=True)
+
+        # Set up the tokenizer for targets
+        with tokenizer.as_target_tokenizer():
+            labels = tokenizer(targets, truncation=True)
+
+        model_inputs["labels"] = labels["input_ids"]
+        return model_inputs
+
+    tokenized_dataset = dataset.map(
+        preprocess_function,
+        batched=True,
+        remove_columns=dataset["train"].column_names,
+    )
+
+    # 4. evaluation metrics
+    def compute_metrics(eval_preds):
+        preds = eval_preds.predictions
+        labels = eval_preds.label_ids
+        if isinstance(preds, tuple):
+            preds = preds[0]
+        print(preds.shape)
+        if len(preds.shape) == 3:
+            preds = preds.argmax(axis=-1)
+
+        decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)
+        # Replace -100 in the labels as we can't decode them.
+        labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
+        decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
+
+        # Some simple post-processing
+        decoded_preds = [a.strip().split(';') for a in decoded_preds]
+        decoded_labels = [a.strip().split(';') for a in decoded_labels]
+
+        precs, recalls, f_scores = [], [], []
+        num_match, num_pred, num_gold = [], [], []
+        for pred, label in zip(decoded_preds, decoded_labels):
+            pred_set = set(pred)
+            label_set = set(label)
+            match_set = label_set.intersection(pred_set)
+            p = float(len(match_set)) / float(len(pred_set)) if len(pred_set) > 0 else 0.0
+            r = float(len(match_set)) / float(len(label_set)) if len(label_set) > 0 else 0.0
+            f1 = float(2 * (p * r)) / (p + r) if (p + r) > 0 else 0.0
+            precs.append(p)
+            recalls.append(r)
+            f_scores.append(f1)
+            num_match.append(len(match_set))
+            num_pred.append(len(pred_set))
+            num_gold.append(len(label_set))
+
+            # print(f'raw_PRED: {raw_pred}')
+            print(f'PRED: num={len(pred_set)} - {pred_set}')
+            print(f'GT: num={len(label_set)} - {label_set}')
+            print(f'p={p}, r={r}, f1={f1}')
+            print('-' * 20)
+
+        result = {
+            'precision@M': np.mean(precs) * 100.0,
+            'recall@M': np.mean(recalls) * 100.0,
+            'fscore@M': np.mean(f_scores) * 100.0,
+            'num_match': np.mean(num_match),
+            'num_pred': np.mean(num_pred),
+            'num_gold': np.mean(num_gold),
+        }
+
+        result = {k: round(v, 2) for k, v in result.items()}
+        return result
+
+    # 5. train
+    from transformers import DataCollatorForSeq2Seq, Seq2SeqTrainingArguments, Seq2SeqTrainer
+
+    data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)
+
+    model_name = 'KeyBartAdapter'
+
+    args = Seq2SeqTrainingArguments(
+        model_name,
+        evaluation_strategy="epoch",
+        save_strategy="epoch",
+        learning_rate=2e-5,
+        per_device_train_batch_size=train_batch_size,
+        per_device_eval_batch_size=eval_batch_size,
+        weight_decay=0.01,
+        save_total_limit=3,
+        num_train_epochs=num_epoch,
+        logging_steps=4,
+        load_best_model_at_end=True,
+        metric_for_best_model='fscore@M',
+        predict_with_generate=True,
+        fp16=torch.cuda.is_available(),  # speeds up training on modern GPUs.
+        # eval_accumulation_steps=10,
+    )
+
+    trainer = Seq2SeqTrainer(
+        model,
+        args,
+        train_dataset=tokenized_dataset["train"],
+        eval_dataset=tokenized_dataset["train"],
+        data_collator=data_collator,
+        tokenizer=tokenizer,
+        compute_metrics=compute_metrics
+    )
+
+    trainer.train()
+
+    # 6. push
+    if push_to_hub:
+        commit_msg = f'{model_name}_{num_epoch}'
+        tokenizer.push_to_hub(commit_message=commit_msg, repo_id=model_name)
+        model.push_to_hub(commit_message=commit_msg, repo_id=model_name)
+
+    return model, tokenizer
+
+if __name__ == '__main__':
+    import sys
+    from pathlib import Path
+    project_root = Path(__file__).parent.parent.parent.absolute()
+    sys.path.append(project_root.__str__())
+
+
+    # code
+    import argparse
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("--epoch", help="number of epochs", default=30)
+    parser.add_argument("--train_batch_size", help="training batch size", default=16)
+    parser.add_argument("--eval_batch_size", help="evaluation batch size", default=16)
+    parser.add_argument("--push", help="whether push the model to hub", action='store_true')
+
+    args = parser.parse_args()
+    print(args)
+
+    model, tokenizer = train(
+        push_to_hub= bool(args.push),
+        num_epoch= int(args.epoch),
+        train_batch_size= int(args.train_batch_size),
+        eval_batch_size= int(args.eval_batch_size)
+    )
+

setup.py

@@ -0,0 +1,23 @@
+from setuptools import setup, find_packages
+
+with open("README.md", "r") as readme_file:
+    readme = readme_file.read()
+
+requirements = ["sentence_transformers"]
+
+setup(
+    name="LiteratureResearchTool",
+    version="0.4.0",
+    author="Tao Xiang",
+    author_email="tao.xiang@tum.de",
+    description="A tool for literature research and analysis",
+    long_description=readme,
+    long_description_content_type="text/markdown",
+    url="https://github.com/Mondkuchen/idp_LiteratureResearch_Tool",
+    packages=find_packages(),
+    install_requires=requirements,
+    classifiers=[
+  "Programming Language :: Python :: 3.7",
+  "License :: OSI Approved :: MIT License",
+    ],
+)