second commit

README.md

@@ -1,3 +1,143 @@
----
-license: mit
----
+---
+language: ja
+thumbnail: https://github.com/ycat3/japanese-pretrained-models/blob/master/jweb.png
+tags:
+- ja
+- japanese
+- gpt2
+- text-generation
+- lm
+- nlp
+- rust
+- rust-bert
+
+license: mit
+
+datasets:
+- cc100
+- wikipedia
+- AozoraBunko
+
+widget:
+- text: "夏目漱石は、"
+---
+
+# japanese-soseki-gpt2-1b
+
+![jweb-icon](./jweb.png)
+
+This repository provides a 1.3B-parameter finetuned Japanese GPT2 model.
+The model was finetuned by [jweb](https://jweb.asia/) based on trained by [rinna Co., Ltd.](https://corp.rinna.co.jp/)
+Both pytorch(pytorch_model.bin) and Rust(rust_model.ot) models are provided
+
+# How to use the model
+
+*NOTE:* Use `T5Tokenizer` to initiate the tokenizer.
+
+python
+~~~~
+import torch
+from transformers import T5Tokenizer, AutoModelForCausalLM
+
+tokenizer = T5Tokenizer.from_pretrained("jweb/japanese-soseki-gpt2-1b")
+model = AutoModelForCausalLM.from_pretrained("jweb/japanese-soseki-gpt2-1b")
+
+if torch.cuda.is_available():
+    model = model.to("cuda")
+
+text = "夏目漱石は、"
+token_ids = tokenizer.encode(text, add_special_tokens=False, return_tensors="pt")
+
+with torch.no_grad():
+    output_ids = model.generate(
+        token_ids.to(model.device),
+        max_length=128,
+        min_length=40,
+        do_sample=True,
+        repetition_penalty= 1.6,
+        early_stopping= True,
+        num_beams= 5,
+        temperature= 1.0,
+        top_k=500,
+        top_p=0.95,
+        pad_token_id=tokenizer.pad_token_id,
+        bos_token_id=tokenizer.bos_token_id,
+        eos_token_id=tokenizer.eos_token_id,
+    )
+
+output = tokenizer.decode(output_ids.tolist()[0])
+print(output)  
+# sample output: 夏目漱石は、明治時代を代表する文豪です。夏目漱石の代表作は「吾輩は猫である」や「坊っちゃん」、「草枕」「三四郎」、それに「虞美人草(ぐびじんそう)」などたくさんあります。
+~~~~
+
+rust
+~~~~
+use rust_bert::gpt2::GPT2Generator;
+use rust_bert::pipelines::common::{ModelType, TokenizerOption};
+use rust_bert::pipelines::generation_utils::{GenerateConfig, LanguageGenerator};
+use rust_bert::resources::{ RemoteResource,  ResourceProvider};
+use tch::Device;
+
+fn main() -> anyhow::Result<()> {
+    let model_resource = Box::new(RemoteResource {     
+        url: "https://huggingface.co/jweb/japanese-soseki-gpt2-1b/resolve/main/rust_model.ot".into(),
+        cache_subdir: "japanese-soseki-gpt2-1b/model".into(),        
+    });
+    let config_resource = Box::new(RemoteResource {     
+        url: "https://huggingface.co/jweb/japanese-soseki-gpt2-1b/resolve/main/config.json".into(),
+        cache_subdir: "japanese-soseki-gpt2-1b/config".into(),        
+    });
+    let vocab_resource = Box::new(RemoteResource {     
+        url: "https://huggingface.co/jweb/japanese-soseki-gpt2-1b/resolve/main/spiece.model".into(),
+        cache_subdir: "japanese-soseki-gpt2-1b/vocab".into(),        
+    });
+    let vocab_resource_token = vocab_resource.clone();
+    let merges_resource = vocab_resource.clone();    
+    let generate_config = GenerateConfig {        
+        model_resource,
+        config_resource,
+        vocab_resource,
+        merges_resource, // not used        
+        device: Device::Cpu,
+        repetition_penalty: 1.6,
+        min_length: 40,
+        max_length: 128,
+        do_sample: true,
+        early_stopping: true,
+        num_beams: 5,
+        temperature: 1.0,
+        top_k: 500,
+        top_p: 0.95,
+        ..Default::default()
+    };
+    let tokenizer = TokenizerOption::from_file(
+        ModelType::T5,
+        vocab_resource_token.get_local_path().unwrap().to_str().unwrap(),
+        None,
+        true,
+        None,
+        None,
+    )?;
+    let mut gpt2_model = GPT2Generator::new_with_tokenizer(generate_config, tokenizer.into())?;
+    gpt2_model.set_device(Device::cuda_if_available());
+    let input_text = "夏目漱石は、";
+    let t1 = std::time::Instant::now();
+    let output = gpt2_model.generate(Some(&[input_text]), None);
+    println!("{}", output[0].text);
+    println!("Elapsed Time(ms):{}",t1.elapsed().as_millis()); 
+    Ok(())
+}
+// sample output: 夏目漱石は、明治から大正にかけて活躍した日本の小説家です。彼は「吾輩は猫である」や「坊っちゃん」、「草枕」「三四郎」、あるいは「虞美人草」などの小説で知られていますが、「明暗」のような小説も書いていました。
+~~~~
+
+# Model architecture
+A 24-layer, 2048-hidden-size transformer-based language model.
+
+# Training
+The model was trained on [Japanese C4](https://huggingface.co/datasets/allenai/c4), [Japanese CC-100](http://data.statmt.org/cc-100/ja.txt.xz) and [Japanese Wikipedia](https://dumps.wikimedia.org/other/cirrussearch) to optimize a traditional language modelling objective. It reaches around 14 perplexity on a chosen validation set from the same data.
+# Finetuning
+The model was finetuned on [Aozorabunko](https://github.com/aozorabunko/aozorabunko), especially Natume Soseki books.
+# Tokenization
+The model uses a [sentencepiece](https://github.com/google/sentencepiece)-based tokenizer. The vocabulary was first trained on a selected subset from the training data using the official sentencepiece training script, and then augmented with emojis and symbols.
+# Licenese
+[The MIT license](https://opensource.org/licenses/MIT)

tokenizer.json/added_tokens.json

@@ -0,0 +1,6 @@
+{
+  "[CLS]": 44878,
+  "[MASK]": 44879,
+  "[PAD]": 44877,
+  "[SEP]": 44876
+}

tokenizer.json/special_tokens_map.json

@@ -0,0 +1,9 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "[CLS]",
+  "eos_token": "</s>",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "<unk>"
+}

tokenizer.json/spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d955d3e358f66e9e8320bd834524b1264c21cc66a68fb18f3a4f091ed25a5c40
+size 1044749

tokenizer.json/tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "</s>",
+  "extra_ids": 0,
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "sp_model_kwargs": {},
+  "special_tokens_map_file": "/home/mycat/model/rinna/rinna-gpt/special_tokens_map.json",
+  "tokenizer_class": "T5Tokenizer",
+  "unk_token": "<unk>"
+}