multimodal/tools/instruct_tuning_data/merge.py

import os
import glob
import random
OUT_DIR = "/gpfs/u/home/LMCG/LMCGljnn/scratch-shared/junyan/raw/instruct/all"


if __name__ == "__main__":
    tars = glob.glob(os.path.join("/gpfs/u/home/LMCG/LMCGljnn/scratch-shared/junyan/raw/instruct", "*", "*.tar"))
    random.shuffle(tars)
    os.makedirs(OUT_DIR, exist_ok=True)
    for i, tar in enumerate(tars):
        dst = os.path.join(OUT_DIR, f"{str(i).zfill(6)}.tar")
        os.symlink(tar, dst)