update

iGPT/models/husky.py

@@ -66,13 +66,10 @@ def load_model(
 ):
     kwargs = {"torch_dtype": torch.float16}
 
-    if not os.path.exists(model_path[1]):
-        apply_delta(model_path[0], model_path[1], model_path[2])
-
     tokenizer = AutoTokenizer.from_pretrained(
-        model_path[1], use_fast=False)
+        model_path, use_fast=False)
     model = Blip2LlaMAForConditionalGeneration.from_pretrained(
-        model_path[1], low_cpu_mem_usage=True, **kwargs
+        model_path, low_cpu_mem_usage=True, **kwargs
     )
 
     if load_8bit:
@@ -337,12 +334,41 @@ class Chat:
         else:
             self.conv = get_default_conv_template(self.model_path).copy()
 
+
+def download_if_not_exists(base_path, delta_path, new_path):
+    if os.path.exists(new_path):
+        return
+    
+    if not os.path.exists(base_path):
+        # download if not exists
+        os.system('bash third-party/llama_download.sh')
+        
+    output_dir = os.path.join(os.path.dirname(base_path), 'llama_7B_hf')
+    
+    if not os.path.exists(output_dir):
+        # convert to hf format if not exists
+        from .husky_src.convert_llama_weights_to_hf import write_model, write_tokenizer
+        write_model(
+            model_path=output_dir,
+            input_base_path=os.path.join(base_path, '7B'),
+            model_size="7B",
+        )
+        spm_path = os.path.join(base_path, "tokenizer.model")
+        write_tokenizer(output_dir, spm_path)
+
+    apply_delta(output_dir, new_path, delta_path)
+
+
 class HuskyVQA:
     def __init__(
         self,
         device
     ):
-        model_path=["model_zoo/llama-7b-hf", "model_zoo/husky-7b-v0_01", 'model_zoo/husky-7b-delta-v0_01']
+        model_path = 'model_zoo/husky-7b-v0_01'
+        download_if_not_exists(base_path="model_zoo/llama", 
+                               delta_path="model_zoo/husky-7b-delta-v0_01",
+                               new_path=model_path)
+        
         load_8bit=True
         max_new_tokens=512
         self.chat = Chat(

iGPT/models/husky_src/convert_llama_weights_to_hf.py

@@ -0,0 +1,281 @@
+# Copyright 2022 EleutherAI and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/convert_llama_weights_to_hf.py
+
+import argparse
+import gc
+import json
+import math
+import os
+import shutil
+import warnings
+
+import torch
+
+from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer
+
+
+try:
+    from transformers import LlamaTokenizerFast
+except ImportError as e:
+    warnings.warn(e)
+    warnings.warn(
+        "The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"
+    )
+    LlamaTokenizerFast = None
+
+"""
+Sample usage:
+
+```
+python src/transformers/models/llama/convert_llama_weights_to_hf.py \
+    --input_dir /path/to/downloaded/llama/weights --model_size 7B --output_dir /output/path
+```
+
+Thereafter, models can be loaded via:
+
+```py
+from transformers import LlamaForCausalLM, LlamaTokenizer
+
+model = LlamaForCausalLM.from_pretrained("/output/path")
+tokenizer = LlamaTokenizer.from_pretrained("/output/path")
+```
+
+Important note: you need to be able to host the whole model in RAM to execute this script (even if the biggest versions
+come in several checkpoints they each contain a part of each weight of the model, so we need to load them all in RAM).
+"""
+
+INTERMEDIATE_SIZE_MAP = {
+    "7B": 11008,
+    "13B": 13824,
+    "30B": 17920,
+    "65B": 22016,
+}
+NUM_SHARDS = {
+    "7B": 1,
+    "13B": 2,
+    "30B": 4,
+    "65B": 8,
+}
+
+
+def compute_intermediate_size(n):
+    return int(math.ceil(n * 8 / 3) + 255) // 256 * 256
+
+
+def read_json(path):
+    with open(path, "r") as f:
+        return json.load(f)
+
+
+def write_json(text, path):
+    with open(path, "w") as f:
+        json.dump(text, f)
+
+
+def write_model(model_path, input_base_path, model_size):
+    os.makedirs(model_path, exist_ok=True)
+    tmp_model_path = os.path.join(model_path, "tmp")
+    os.makedirs(tmp_model_path, exist_ok=True)
+
+    params = read_json(os.path.join(input_base_path, "params.json"))
+    num_shards = NUM_SHARDS[model_size]
+    n_layers = params["n_layers"]
+    n_heads = params["n_heads"]
+    n_heads_per_shard = n_heads // num_shards
+    dim = params["dim"]
+    dims_per_head = dim // n_heads
+    base = 10000.0
+    inv_freq = 1.0 / (base ** (torch.arange(0, dims_per_head, 2).float() / dims_per_head))
+
+    # permute for sliced rotary
+    def permute(w):
+        return w.view(n_heads, dim // n_heads // 2, 2, dim).transpose(1, 2).reshape(dim, dim)
+
+    print(f"Fetching all parameters from the checkpoint at {input_base_path}.")
+    # Load weights
+    if model_size == "7B":
+        # Not sharded
+        # (The sharded implementation would also work, but this is simpler.)
+        loaded = torch.load(os.path.join(input_base_path, "consolidated.00.pth"), map_location="cpu")
+    else:
+        # Sharded
+        loaded = [
+            torch.load(os.path.join(input_base_path, f"consolidated.{i:02d}.pth"), map_location="cpu")
+            for i in range(num_shards)
+        ]
+    param_count = 0
+    index_dict = {"weight_map": {}}
+    for layer_i in range(n_layers):
+        filename = f"pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"
+        if model_size == "7B":
+            # Unsharded
+            state_dict = {
+                f"model.layers.{layer_i}.self_attn.q_proj.weight": permute(
+                    loaded[f"layers.{layer_i}.attention.wq.weight"]
+                ),
+                f"model.layers.{layer_i}.self_attn.k_proj.weight": permute(
+                    loaded[f"layers.{layer_i}.attention.wk.weight"]
+                ),
+                f"model.layers.{layer_i}.self_attn.v_proj.weight": loaded[f"layers.{layer_i}.attention.wv.weight"],
+                f"model.layers.{layer_i}.self_attn.o_proj.weight": loaded[f"layers.{layer_i}.attention.wo.weight"],
+                f"model.layers.{layer_i}.mlp.gate_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w1.weight"],
+                f"model.layers.{layer_i}.mlp.down_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w2.weight"],
+                f"model.layers.{layer_i}.mlp.up_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w3.weight"],
+                f"model.layers.{layer_i}.input_layernorm.weight": loaded[f"layers.{layer_i}.attention_norm.weight"],
+                f"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[f"layers.{layer_i}.ffn_norm.weight"],
+            }
+        else:
+            # Sharded
+            # Note that in the 13B checkpoint, not cloning the two following weights will result in the checkpoint
+            # becoming 37GB instead of 26GB for some reason.
+            state_dict = {
+                f"model.layers.{layer_i}.input_layernorm.weight": loaded[0][
+                    f"layers.{layer_i}.attention_norm.weight"
+                ].clone(),
+                f"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[0][
+                    f"layers.{layer_i}.ffn_norm.weight"
+                ].clone(),
+            }
+            state_dict[f"model.layers.{layer_i}.self_attn.q_proj.weight"] = permute(
+                torch.cat(
+                    [
+                        loaded[i][f"layers.{layer_i}.attention.wq.weight"].view(n_heads_per_shard, dims_per_head, dim)
+                        for i in range(num_shards)
+                    ],
+                    dim=0,
+                ).reshape(dim, dim)
+            )
+            state_dict[f"model.layers.{layer_i}.self_attn.k_proj.weight"] = permute(
+                torch.cat(
+                    [
+                        loaded[i][f"layers.{layer_i}.attention.wk.weight"].view(n_heads_per_shard, dims_per_head, dim)
+                        for i in range(num_shards)
+                    ],
+                    dim=0,
+                ).reshape(dim, dim)
+            )
+            state_dict[f"model.layers.{layer_i}.self_attn.v_proj.weight"] = torch.cat(
+                [
+                    loaded[i][f"layers.{layer_i}.attention.wv.weight"].view(n_heads_per_shard, dims_per_head, dim)
+                    for i in range(num_shards)
+                ],
+                dim=0,
+            ).reshape(dim, dim)
+
+            state_dict[f"model.layers.{layer_i}.self_attn.o_proj.weight"] = torch.cat(
+                [loaded[i][f"layers.{layer_i}.attention.wo.weight"] for i in range(num_shards)], dim=1
+            )
+            state_dict[f"model.layers.{layer_i}.mlp.gate_proj.weight"] = torch.cat(
+                [loaded[i][f"layers.{layer_i}.feed_forward.w1.weight"] for i in range(num_shards)], dim=0
+            )
+            state_dict[f"model.layers.{layer_i}.mlp.down_proj.weight"] = torch.cat(
+                [loaded[i][f"layers.{layer_i}.feed_forward.w2.weight"] for i in range(num_shards)], dim=1
+            )
+            state_dict[f"model.layers.{layer_i}.mlp.up_proj.weight"] = torch.cat(
+                [loaded[i][f"layers.{layer_i}.feed_forward.w3.weight"] for i in range(num_shards)], dim=0
+            )
+
+        state_dict[f"model.layers.{layer_i}.self_attn.rotary_emb.inv_freq"] = inv_freq
+        for k, v in state_dict.items():
+            index_dict["weight_map"][k] = filename
+            param_count += v.numel()
+        torch.save(state_dict, os.path.join(tmp_model_path, filename))
+
+    filename = f"pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"
+    if model_size == "7B":
+        # Unsharded
+        state_dict = {
+            "model.embed_tokens.weight": loaded["tok_embeddings.weight"],
+            "model.norm.weight": loaded["norm.weight"],
+            "lm_head.weight": loaded["output.weight"],
+        }
+    else:
+        state_dict = {
+            "model.norm.weight": loaded[0]["norm.weight"],
+            "model.embed_tokens.weight": torch.cat(
+                [loaded[i]["tok_embeddings.weight"] for i in range(num_shards)], dim=1
+            ),
+            "lm_head.weight": torch.cat([loaded[i]["output.weight"] for i in range(num_shards)], dim=0),
+        }
+
+    for k, v in state_dict.items():
+        index_dict["weight_map"][k] = filename
+        param_count += v.numel()
+    torch.save(state_dict, os.path.join(tmp_model_path, filename))
+
+    # Write configs
+    index_dict["metadata"] = {"total_size": param_count * 2}
+    write_json(index_dict, os.path.join(tmp_model_path, "pytorch_model.bin.index.json"))
+
+    config = LlamaConfig(
+        hidden_size=dim,
+        intermediate_size=compute_intermediate_size(dim),
+        num_attention_heads=params["n_heads"],
+        num_hidden_layers=params["n_layers"],
+        rms_norm_eps=params["norm_eps"],
+    )
+    config.save_pretrained(tmp_model_path)
+
+    # Make space so we can load the model properly now.
+    del state_dict
+    del loaded
+    gc.collect()
+
+    print("Loading the checkpoint in a Llama model.")
+    model = LlamaForCausalLM.from_pretrained(tmp_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True)
+    # Avoid saving this as part of the config.
+    del model.config._name_or_path
+
+    print("Saving in the Transformers format.")
+    model.save_pretrained(model_path)
+    shutil.rmtree(tmp_model_path)
+
+
+def write_tokenizer(tokenizer_path, input_tokenizer_path):
+    # Initialize the tokenizer based on the `spm` model
+    tokenizer_class = LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
+    print(f"Saving a {tokenizer_class.__name__} to {tokenizer_path}.")
+    tokenizer = tokenizer_class(input_tokenizer_path)
+    tokenizer.save_pretrained(tokenizer_path)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--input_dir",
+        help="Location of LLaMA weights, which contains tokenizer.model and model folders",
+    )
+    parser.add_argument(
+        "--model_size",
+        choices=["7B", "13B", "30B", "65B", "tokenizer_only"],
+    )
+    parser.add_argument(
+        "--output_dir",
+        help="Location to write HF model and tokenizer",
+    )
+    args = parser.parse_args()
+    if args.model_size != "tokenizer_only":
+        write_model(
+            model_path=args.output_dir,
+            input_base_path=os.path.join(args.input_dir, args.model_size),
+            model_size=args.model_size,
+        )
+    spm_path = os.path.join(args.input_dir, "tokenizer.model")
+    write_tokenizer(args.output_dir, spm_path)
+
+
+if __name__ == "__main__":
+    main()

iGPT/models/inpainting.py

@@ -8,6 +8,7 @@ from .utils import gen_new_name, prompts
 import torch
 from omegaconf import OmegaConf
 import numpy as np
+import wget
 from .inpainting_src.ldm_inpainting.ldm.models.diffusion.ddim import DDIMSampler
 from .inpainting_src.ldm_inpainting.ldm.util import instantiate_from_config
 from .utils import cal_dilate_factor, dilate_mask
@@ -35,16 +36,22 @@ def make_batch(image, mask, device):
 
 class LDMInpainting:
     def __init__(self, device):
-        ckpt_path = 'model_zoo/ldm_inpainting_big.ckpt'
+        self.model_checkpoint_path = 'model_zoo/ldm_inpainting_big.ckpt'
         config = './iGPT/models/inpainting_src/ldm_inpainting/config.yaml'
         self.ddim_steps = 50
         self.device = device
         config = OmegaConf.load(config)
         model = instantiate_from_config(config.model)
-        model.load_state_dict(torch.load(ckpt_path)["state_dict"], strict=False)
+        self.download_parameters()
+        model.load_state_dict(torch.load(self.model_checkpoint_path)["state_dict"], strict=False)
         self.model = model.to(device=device)
         self.sampler = DDIMSampler(model)
     
+    def download_parameters(self):
+        url = 'https://heibox.uni-heidelberg.de/f/4d9ac7ea40c64582b7c9/?dl=1'
+        if not os.path.exists(self.model_checkpoint_path):
+            wget.download(url, out=self.model_checkpoint_path)
+
     @prompts(name="Remove the Masked Object",
              description="useful when you want to remove an object by masking the region in the image. "
                          "like: remove masked object or inpaint the masked region.. "

iGPT/models/utils.py

@@ -38,11 +38,12 @@ def gen_new_name(orginal_name, suffix="update", ext="png"):
     name_split = os.path.splitext(filename)[0].split('_')
     this_new_uuid = str(uuid.uuid4())[:3]
     timestamp = int(math.modf(time.time())[0] * 1000)
-    if len(name_split) == 1:
-        prev_file_name = name_split[0]
-    else:
-        # assert len(name_split) == 3
-        prev_file_name = name_split[1]
+    prev_file_name = name_split[0]
+    # if len(name_split) == 1:
+    #     prev_file_name = name_split[0]
+    # else:
+    #     # assert len(name_split) == 3
+    #     prev_file_name = name_split[0]
     if len(suffix.strip()) == 0:
         new_file_name = f'{this_new_uuid}{timestamp:03d}_{prev_file_name}.{ext}'
     else:

requirements.txt

@@ -23,4 +23,5 @@ kornia==0.5.0
 sentencepiece==0.1.98
 accelerate==0.18.0
 timm==0.6.13
-git+https://github.com/facebookresearch/segment-anything.git
+git+https://github.com/facebookresearch/segment-anything.git
+wget

third-party/llama_download.sh

@@ -0,0 +1,33 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# This software may be used and distributed according to the terms of the GNU General Public License version 3.
+
+PRESIGNED_URL=""             # replace with presigned url from email
+MODEL_SIZE="7B"  # edit this list with the model sizes you wish to download
+TARGET_FOLDER="model_zoo/llama"             # where all files should end up
+
+declare -A N_SHARD_DICT
+
+N_SHARD_DICT["7B"]="0"
+N_SHARD_DICT["13B"]="1"
+N_SHARD_DICT["30B"]="3"
+N_SHARD_DICT["65B"]="7"
+
+echo "Downloading tokenizer"
+wget ${PRESIGNED_URL/'*'/"tokenizer.model"} -O ${TARGET_FOLDER}"/tokenizer.model"
+wget ${PRESIGNED_URL/'*'/"tokenizer_checklist.chk"} -O ${TARGET_FOLDER}"/tokenizer_checklist.chk"
+
+(cd ${TARGET_FOLDER} && md5sum -c tokenizer_checklist.chk)
+
+for i in ${MODEL_SIZE//,/ }
+do
+    echo "Downloading ${i}"
+    mkdir -p ${TARGET_FOLDER}"/${i}"
+    for s in $(seq -f "0%g" 0 ${N_SHARD_DICT[$i]})
+    do
+        wget ${PRESIGNED_URL/'*'/"${i}/consolidated.${s}.pth"} -O ${TARGET_FOLDER}"/${i}/consolidated.${s}.pth"
+    done
+    wget ${PRESIGNED_URL/'*'/"${i}/params.json"} -O ${TARGET_FOLDER}"/${i}/params.json"
+    wget ${PRESIGNED_URL/'*'/"${i}/checklist.chk"} -O ${TARGET_FOLDER}"/${i}/checklist.chk"
+    echo "Checking checksums"
+    (cd ${TARGET_FOLDER}"/${i}" && md5sum -c checklist.chk)
+done