data_config.json

{
    "text2reddit": {
        "weight": 30,
        "files": {"/home/reddit/reddit-title-body/reddit_title_text_2010.jsonl.gz": 1,
             "/home/reddit/reddit-title-body/reddit_title_text_2011.jsonl.gz":  1,
             "/home/reddit/reddit-title-body/reddit_title_text_2012.jsonl.gz":  3,
             "/home/reddit/reddit-title-body/reddit_title_text_2013.jsonl.gz":  5,
             "/home/reddit/reddit-title-body/reddit_title_text_2014.jsonl.gz":  8,
             "/home/reddit/reddit-title-body/reddit_title_text_2015.jsonl.gz":  11,
             "/home/reddit/reddit-title-body/reddit_title_text_2016.jsonl.gz":  12,
             "/home/reddit/reddit-title-body/reddit_title_text_2017.jsonl.gz":  13,
             "/home/reddit/reddit-title-body/reddit_title_text_2018.jsonl.gz":  15,
             "/home/reddit/reddit-title-body/reddit_title_text_2019.jsonl.gz":  19,
             "/home/reddit/reddit-title-body/reddit_title_text_2020.jsonl.gz":  23,
             "/home/reddit/reddit-title-body/reddit_title_text_2021.jsonl.gz":  12
             }
        
    },
    "question2title": {
        "weight": 30,
        "files": {"/home/sbert_pretrained_models/datasets/embedding-training-data/yahoo_answers_title_question.jsonl.gz": 40,
            "/home/stackexchange_archive/jsonl/skeptics.stackexchange.com.jsonl.gz": 1, "/home/stackexchange_archive/jsonl/writers.stackexchange.com.jsonl.gz": 1, "/home/stackexchange_archive/jsonl/astronomy.stackexchange.com.jsonl.gz": 1, "/home/stackexchange_archive/jsonl/vi.stackexchange.com.jsonl.gz": 1, "/home/stackexchange_archive/jsonl/cstheory.stackexchange.com.jsonl.gz": 1, "/home/stackexchange_archive/jsonl/engineering.stackexchange.com.jsonl.gz": 1, "/home/stackexchange_archive/jsonl/french.stackexchange.com.jsonl.gz": 1, "/home/stackexchange_archive/jsonl/economics.stackexchange.com.jsonl.gz": 1, "/home/stackexchange_archive/jsonl/anime.stackexchange.com.jsonl.gz": 1, "/home/stackexchange_archive/jsonl/islam.stackexchange.com.jsonl.gz": 1, "/home/stackexchange_archive/jsonl/expressionengine.stackexchange.com.jsonl.gz": 1, "/home/stackexchange_archive/jsonl/politics.stackexchange.com.jsonl.gz": 1, "/home/stackexchange_archive/jsonl/history.stackexchange.com.jsonl.gz": 1, "/home/stackexchange_archive/jsonl/christianity.stackexchange.com.jsonl.gz": 1, "/home/stackexchange_archive/jsonl/boardgames.stackexchange.com.jsonl.gz": 1, "/home/stackexchange_archive/jsonl/civicrm.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/jsonl/craftcms.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/jsonl/hinduism.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/jsonl/networkengineering.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/jsonl/german.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/jsonl/philosophy.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/jsonl/gardening.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/jsonl/space.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/jsonl/bicycles.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/jsonl/quant.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/jsonl/puzzling.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/jsonl/law.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/jsonl/arduino.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/jsonl/aviation.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/jsonl/softwarerecs.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/jsonl/movies.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/jsonl/music.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/jsonl/emacs.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/jsonl/dsp.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/jsonl/japanese.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/jsonl/mechanics.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/jsonl/crypto.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/jsonl/cooking.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/jsonl/photo.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/jsonl/workplace.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/jsonl/biology.stackexchange.com.jsonl.gz": 3, "/home/stackexchange_archive/jsonl/bitcoin.stackexchange.com.jsonl.gz": 3, "/home/stackexchange_archive/jsonl/worldbuilding.stackexchange.com.jsonl.gz": 3, "/home/stackexchange_archive/jsonl/datascience.stackexchange.com.jsonl.gz": 3, "/home/stackexchange_archive/jsonl/ux.stackexchange.com.jsonl.gz": 3, "/home/stackexchange_archive/jsonl/webapps.stackexchange.com.jsonl.gz": 3, "/home/stackexchange_archive/jsonl/graphicdesign.stackexchange.com.jsonl.gz": 3, "/home/stackexchange_archive/jsonl/raspberrypi.stackexchange.com.jsonl.gz": 3, "/home/stackexchange_archive/jsonl/money.stackexchange.com.jsonl.gz": 3, "/home/stackexchange_archive/jsonl/judaism.stackexchange.com.jsonl.gz": 3, "/home/stackexchange_archive/jsonl/ethereum.stackexchange.com.jsonl.gz": 3, "/home/stackexchange_archive/jsonl/academia.stackexchange.com.jsonl.gz": 3, "/home/stackexchange_archive/jsonl/chemistry.stackexchange.com.jsonl.gz": 3, "/home/stackexchange_archive/jsonl/webmasters.stackexchange.com.jsonl.gz": 3, "/home/stackexchange_archive/jsonl/meta.stackoverflow.com.jsonl.gz": 3, "/home/stackexchange_archive/jsonl/cs.stackexchange.com.jsonl.gz": 4, "/home/stackexchange_archive/jsonl/travel.stackexchange.com.jsonl.gz": 4, "/home/stackexchange_archive/jsonl/rpg.stackexchange.com.jsonl.gz": 4, "/home/stackexchange_archive/jsonl/codereview.stackexchange.com.jsonl.gz": 4, "/home/stackexchange_archive/jsonl/gamedev.stackexchange.com.jsonl.gz": 4, "/home/stackexchange_archive/jsonl/android.stackexchange.com.jsonl.gz": 5, "/home/stackexchange_archive/jsonl/softwareengineering.stackexchange.com.jsonl.gz": 5, "/home/stackexchange_archive/jsonl/security.stackexchange.com.jsonl.gz": 5, "/home/stackexchange_archive/jsonl/diy.stackexchange.com.jsonl.gz": 5, "/home/stackexchange_archive/jsonl/scifi.stackexchange.com.jsonl.gz": 6, "/home/stackexchange_archive/jsonl/mathematica.stackexchange.com.jsonl.gz": 7, "/home/stackexchange_archive/jsonl/drupal.stackexchange.com.jsonl.gz": 7, "/home/stackexchange_archive/jsonl/blender.stackexchange.com.jsonl.gz": 7, "/home/stackexchange_archive/jsonl/dba.stackexchange.com.jsonl.gz": 7, "/home/stackexchange_archive/jsonl/ell.stackexchange.com.jsonl.gz": 7, "/home/stackexchange_archive/jsonl/meta.stackexchange.com.jsonl.gz": 7, "/home/stackexchange_archive/jsonl/gaming.stackexchange.com.jsonl.gz": 8, "/home/stackexchange_archive/jsonl/sharepoint.stackexchange.com.jsonl.gz": 8, "/home/stackexchange_archive/jsonl/magento.stackexchange.com.jsonl.gz": 9, "/home/stackexchange_archive/jsonl/wordpress.stackexchange.com.jsonl.gz": 9, "/home/stackexchange_archive/jsonl/salesforce.stackexchange.com.jsonl.gz": 9, "/home/stackexchange_archive/jsonl/english.stackexchange.com.jsonl.gz": 10, "/home/stackexchange_archive/jsonl/apple.stackexchange.com.jsonl.gz": 10, "/home/stackexchange_archive/jsonl/mathoverflow.net.jsonl.gz": 10, "/home/stackexchange_archive/jsonl/gis.stackexchange.com.jsonl.gz": 11, "/home/stackexchange_archive/jsonl/electronics.stackexchange.com.jsonl.gz": 12, "/home/stackexchange_archive/jsonl/physics.stackexchange.com.jsonl.gz": 15, "/home/stackexchange_archive/jsonl/stats.stackexchange.com.jsonl.gz": 15, "/home/stackexchange_archive/jsonl/unix.stackexchange.com.jsonl.gz": 16, "/home/stackexchange_archive/jsonl/tex.stackexchange.com.jsonl.gz": 17, "/home/stackexchange_archive/jsonl/serverfault.com.jsonl.gz": 23, "/home/stackexchange_archive/jsonl/askubuntu.com.jsonl.gz": 29, "/home/stackexchange_archive/jsonl/superuser.com.jsonl.gz": 36, "/home/stackexchange_archive/jsonl/small_stackexchanges.jsonl.gz": 37, "/home/stackexchange_archive/jsonl/math.stackexchange.com.jsonl.gz": 83, "/home/stackexchange_archive/jsonl/stackoverflow.com-Posts.jsonl.gz": 83
            }
    },
    "answer2question": {
        "weight": 30,
        "files": {"/home/sbert_pretrained_models/datasets/embedding-training-data/yahoo_answers_title_answer.jsonl.gz": 80, "/home/sbert_pretrained_models/datasets/embedding-training-data/amazon-qa.jsonl.gz": 80,
            "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/islam.stackexchange.com.jsonl.gz": 1, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/anime.stackexchange.com.jsonl.gz": 1, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/french.stackexchange.com.jsonl.gz": 1, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/civicrm.stackexchange.com.jsonl.gz": 1, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/expressionengine.stackexchange.com.jsonl.gz": 1, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/history.stackexchange.com.jsonl.gz": 1, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/politics.stackexchange.com.jsonl.gz": 1, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/craftcms.stackexchange.com.jsonl.gz": 1, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/christianity.stackexchange.com.jsonl.gz": 1, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/softwarerecs.stackexchange.com.jsonl.gz": 1, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/boardgames.stackexchange.com.jsonl.gz": 1, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/networkengineering.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/space.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/quant.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/philosophy.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/gardening.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/german.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/bicycles.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/law.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/arduino.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/emacs.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/dsp.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/puzzling.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/movies.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/mechanics.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/aviation.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/biology.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/crypto.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/music.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/datascience.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/japanese.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/bitcoin.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/cooking.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/photo.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/workplace.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/meta.stackoverflow.com.jsonl.gz": 2, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/raspberrypi.stackexchange.com.jsonl.gz": 2, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/webapps.stackexchange.com.jsonl.gz": 3, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/judaism.stackexchange.com.jsonl.gz": 3, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/ethereum.stackexchange.com.jsonl.gz": 3, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/worldbuilding.stackexchange.com.jsonl.gz": 3, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/chemistry.stackexchange.com.jsonl.gz": 3, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/graphicdesign.stackexchange.com.jsonl.gz": 3, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/ux.stackexchange.com.jsonl.gz": 3, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/money.stackexchange.com.jsonl.gz": 3, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/cs.stackexchange.com.jsonl.gz": 3, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/webmasters.stackexchange.com.jsonl.gz": 3, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/academia.stackexchange.com.jsonl.gz": 3, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/travel.stackexchange.com.jsonl.gz": 4, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/android.stackexchange.com.jsonl.gz": 4, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/gamedev.stackexchange.com.jsonl.gz": 4, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/rpg.stackexchange.com.jsonl.gz": 4, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/codereview.stackexchange.com.jsonl.gz": 4, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/softwareengineering.stackexchange.com.jsonl.gz": 5, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/security.stackexchange.com.jsonl.gz": 5, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/diy.stackexchange.com.jsonl.gz": 5, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/blender.stackexchange.com.jsonl.gz": 5, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/scifi.stackexchange.com.jsonl.gz": 5, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/mathematica.stackexchange.com.jsonl.gz": 5, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/meta.stackexchange.com.jsonl.gz": 5, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/drupal.stackexchange.com.jsonl.gz": 6, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/dba.stackexchange.com.jsonl.gz": 6, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/ell.stackexchange.com.jsonl.gz": 7, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/magento.stackexchange.com.jsonl.gz": 7, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/sharepoint.stackexchange.com.jsonl.gz": 7, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/gaming.stackexchange.com.jsonl.gz": 7, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/wordpress.stackexchange.com.jsonl.gz": 7, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/mathoverflow.net.jsonl.gz": 8, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/salesforce.stackexchange.com.jsonl.gz": 8, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/apple.stackexchange.com.jsonl.gz": 8, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/gis.stackexchange.com.jsonl.gz": 9, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/english.stackexchange.com.jsonl.gz": 9, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/stats.stackexchange.com.jsonl.gz": 10, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/electronics.stackexchange.com.jsonl.gz": 11, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/physics.stackexchange.com.jsonl.gz": 12, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/unix.stackexchange.com.jsonl.gz": 13, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/tex.stackexchange.com.jsonl.gz": 15, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/serverfault.com.jsonl.gz": 20, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/askubuntu.com.jsonl.gz": 22, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/superuser.com.jsonl.gz": 30, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/small_stackexchanges.jsonl.gz": 38, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/math.stackexchange.com.jsonl.gz": 83, "/home/stackexchange_archive/stackexchange_extracted/TitleAnswer/stackoverflow.com-Posts.jsonl.gz": 83} 
    },
    "abstract2title": {
        "weight": 10,
        "files": {"/home/sbert_pretrained_models/datasets/embedding-training-data/S2ORC_title_abstract.jsonl.gz": 100}
    },
    "review2title": {
        "weight": 10,
        "files": {"/home/sbert_pretrained_models/datasets/embedding-training-data/amazon_review_2018.jsonl.gz": 100}
    },
    "news2title": {
        "weight": 5,
        "files": {"/home/sbert_pretrained_models/datasets/embedding-training-data/agnews.jsonl.gz": 50, "/home/sbert_pretrained_models/datasets/embedding-training-data/ccnews_title_text.jsonl.gz": 50, "/home/sbert_pretrained_models/datasets/embedding-training-data/xsum.jsonl.gz": 25}
    },
    "text2query": {
        "weight": 10,
        "files":  {"/home/sbert_pretrained_models/datasets/embedding-training-data/msmarco-triplets.jsonl.gz": 33, "/home/sbert_pretrained_models/datasets/embedding-training-data/gooaq_pairs.jsonl.gz": 66, "/home/sbert_pretrained_models/datasets/embedding-training-data/NQ-train_pairs.jsonl.gz": 6} 
    },
    "question2question": {
        "weight": 10,
        "files": {"/home/sbert_pretrained_models/datasets/embedding-training-data/WikiAnswers.jsonl.gz": 95, "/home/sbert_pretrained_models/datasets/embedding-training-data/quora_duplicates.jsonl.gz": 5}
    }

}