Added final FungiCLEF 2024 submission configs.

README.md

@@ -1,30 +1,33 @@
-# Transformer-based Fine-Grained Fungi Classification in an Open-Set Scenario
+# Poison-Aware Open-Set Fungi Classification: Reducing the Risk of Poisonous Confusion
 
-This repository is targeted towards solving the FungiCLEF 2023 (https://huggingface.co/spaces/competitions/FungiCLEF2023) challenge. It is based on MMPreTrain (https://github.com/open-mmlab/mmpretrain).
+This repository is targeted towards solving the FungiCLEF 2024 (https://huggingface.co/spaces/BVRA/FungiCLEF2024) challenge. It is based on MMPreTrain (https://github.com/open-mmlab/mmpretrain).
 
 ## Usage
 
 ### Installation
 
 ```bash
-conda create -n fungi2023 python=3.10 pytorch=2.0.1 torchvision=0.15.2 pytorch-cuda=11.8 -c pytorch -c nvidia
-conda activate fungi2023
-pip install -r requirements.txt
-mim install "mmpretrain==1.0.0rc7"
+conda create -p .conda python=3.10 pytorch=2.3 torchvision pytorch-cuda=12.1 -c pytorch -c nvidia
+conda activate .conda/
+pip install future==1.0.0 tensorboard==2.16.2 pandas==2.2.2
+pip install mmpretrain==1.2.0 mmengine==0.10.4 mmcv==2.2.0 -f https://download.openmmlab.com/mmcv/dist/cu121/torch2.3/index.html
 ```
 
 ### Data
 
-The challenge data has to be downloaded and put into _data/fungiclef2022/_.
+The challenge data has to be downloaded and put into _data/fungi2024/_.
 
 ### Training
 
 ```bash
-bash tools/dist_train.sh configs/swinv2_base_w24_b32x4-fp16_fungi+val_res_384_cb_epochs_6.py 4
+bash tools/dist_train.sh configs/fungi2023/swinv2_base_w24_b16x8-fp16_fungi+val2_res_384_genus-loss_no-unknown-target-zero.py 8
+bash tools/dist_train.sh configs/fungi2023/swinv2_base_w24_b16x8-fp16_fungi+val2_res_384_genus-loss_no-unknown-target-zero_metadata_epochs_4.py 8
 ```
 
-### Inference on pre-trained models
+### Inference
+
+The script _script.py_ performs inference and creates a result file for submission.
 
 ```bash
-python tools/test_generate_result_pre-consensus_tta.py models/swinv2_base_w24_b32x4-fp16_fungi+val_res_384_cb_epochs_6.py models/swinv2_base_w24_b32x4-fp16_fungi+val_res_384_cb_epochs_6_20230524-a251a50a.pth results.csv --threshold 0.2 --no-scores
-```
+python script.py
+```

configs/_base_/datasets/fungi_bs16_swin_384.py

@@ -59,7 +59,7 @@ train_dataloader = dict(
     num_workers=8,
     dataset=dict(
         type=dataset_type,
-        data_root='data/fungi2023/',
+        data_root='data/fungi2024/',
         ann_file='FungiCLEF2023_train_metadata_PRODUCTION.csv',
         data_prefix='DF20/',
         pipeline=train_pipeline),
@@ -71,7 +71,7 @@ val_dataloader = dict(
     num_workers=8,
     dataset=dict(
         type=dataset_type,
-        data_root='data/fungi2023/',
+        data_root='data/fungi2024/',
         ann_file='FungiCLEF2023_val_metadata_PRODUCTION.csv',
         data_prefix='DF21/',
         pipeline=test_pipeline),

configs/fungi2023/swinv2_base_w24_b16x8-fp16_fungi+val2_res_384_genus-loss_no-unknown-target-zero.py

@@ -0,0 +1,133 @@
+_base_ = [
+    '../_base_/models/swin_transformer_v2/base_384_aug.py', '../_base_/datasets/fungi_bs16_swin_384.py',
+    '../_base_/schedules/fungi_bs64_adamw_swin.py', '../_base_/default_runtime.py'
+]
+
+# model settings
+checkpoint = 'https://download.openmmlab.com/mmclassification/v0/swin-v2/pretrain/swinv2-base-w12_3rdparty_in21k-192px_20220803-f7dc9763.pth'  # noqa
+model = dict(
+    backbone=dict(
+        window_size=[24, 24, 24, 12],
+        pretrained_window_sizes=[12, 12, 12, 6],
+        init_cfg=dict(
+            type='Pretrained',
+            checkpoint=checkpoint,
+            prefix='backbone',
+        )),
+    head=dict(
+        type='MultiTaskHead',
+        task_heads=dict(
+            species=dict(
+                type='LinearClsHead',
+                num_classes=1604,),
+            genus=dict(
+                type='LinearClsHead',
+                num_classes=961,)),
+        in_channels=1024,
+        init_cfg=None,  # suppress the default init_cfg of LinearClsHead.
+        loss=dict(
+            type='OpenSetLabelSmoothLoss', label_smooth_val=0.1, mode='original', unknown_target_zero=False),
+        cal_acc=False),
+    train_cfg=dict(_delete_=True),
+)
+
+bgr_mean = [123.675, 116.28, 103.53][::-1]
+bgr_std = [58.395, 57.12, 57.375][::-1]
+
+train_pipeline = [
+    dict(type='LoadImageFromFileFungi'),
+    dict(
+        type='RandomResizedCrop',
+        scale=384,
+        backend='pillow',
+        interpolation='bicubic'),
+    dict(type='RandomFlip', prob=0.5, direction='horizontal'),
+    dict(
+        type='RandAugment',
+        policies='timm_increasing',
+        num_policies=2,
+        total_level=10,
+        magnitude_level=9,
+        magnitude_std=0.5,
+        hparams=dict(
+            pad_val=[round(x) for x in bgr_mean], interpolation='bicubic')),
+    dict(
+        type='RandomErasing',
+        erase_prob=0.25,
+        mode='rand',
+        min_area_ratio=0.02,
+        max_area_ratio=1 / 3,
+        fill_color=bgr_mean,
+        fill_std=bgr_std),
+    dict(
+        type='PackMultiTaskInputs',
+        multi_task_fields=['gt_label']),
+]
+
+test_pipeline = [
+    dict(type='LoadImageFromFileFungi'),
+    dict(
+        type='ResizeEdge',
+        scale=438,
+        edge='short',
+        backend='pillow',
+        interpolation='bicubic'),
+    dict(type='CenterCrop', crop_size=384),
+    dict(
+        type='PackMultiTaskInputs',
+        multi_task_fields=['gt_label']),
+]
+
+train_dataloader = dict(
+    batch_size=16,
+    num_workers=10,
+    dataset=dict(
+        _delete_=True,
+        type='ConcatDataset',
+        datasets=[
+                dict(
+                    type='FungiMultitask',
+                    data_root='data/fungi2024/',
+                    ann_file='FungiCLEF2023_train_metadata_PRODUCTION.csv',
+                    data_prefix='DF20/',
+                    pipeline=train_pipeline),
+                dict(
+                    type='FungiMultitask',
+                    data_root='data/fungi2024/',
+                    ann_file='FungiCLEF2023_val_metadata_PRODUCTION.csv',
+                    data_prefix='DF21/',
+                    pipeline=train_pipeline,
+                    open_set=True),
+            ]))
+
+val_dataloader = dict(
+    batch_size=32,
+    num_workers=10,
+    dataset=dict(
+        type='FungiMultitask',
+        pipeline=test_pipeline))
+val_evaluator = dict(
+    _delete_=True,
+    type='MultiTasksMetric',
+    task_metrics=dict(
+        species=[dict(type='SingleLabelMetric', items=['precision', 'recall', 'f1-score'])],
+        genus=[dict(type='SingleLabelMetric', items=['precision', 'recall', 'f1-score'])],
+))
+
+train_cfg = dict(max_epochs=24)
+
+optim_wrapper = dict(type='AmpOptimWrapper', optimizer=dict(lr=5.e-4 * 64 / 512))
+
+# learning policy
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=0.01,
+        by_epoch=False,
+        end=2100),
+    # main learning rate scheduler
+    dict(type='CosineAnnealingLR', eta_min=0, by_epoch=False, begin=2100)
+]
+
+custom_imports = dict(imports=['mmpretrain_custom'], allow_failed_imports=False)

configs/fungi2023/swinv2_base_w24_b16x8-fp16_fungi+val2_res_384_genus-loss_no-unknown-target-zero_metadata_epochs_4.py

@@ -0,0 +1,132 @@
+_base_ = [
+    '../_base_/models/swin_transformer_v2/base_384_aug.py', '../_base_/datasets/fungi_bs16_swin_384.py',
+    '../_base_/schedules/fungi_bs64_adamw_swin.py', '../_base_/default_runtime.py'
+]
+
+# model settings
+checkpoint = 'https://download.openmmlab.com/mmclassification/v0/swin-v2/pretrain/swinv2-base-w12_3rdparty_in21k-192px_20220803-f7dc9763.pth'  # noqa
+model = dict(
+    backbone=dict(
+        window_size=[24, 24, 24, 12],
+        pretrained_window_sizes=[12, 12, 12, 6],
+        frozen_stages=3,),
+    head=dict(
+        type='MultiTaskHead',
+        task_heads=dict(
+            species=dict(
+                type='MetadataHead',
+                num_classes=1604,
+                data_paths=['data/fungi2024/FungiCLEF2023_train_metadata_PRODUCTION.csv', 'data/fungi2024/FungiCLEF2023_val_metadata_PRODUCTION.csv'],),
+            genus=dict(
+                type='LinearClsHead',
+                num_classes=961,)),
+        in_channels=1024,
+        init_cfg=None,  # suppress the default init_cfg of LinearClsHead.
+        loss=dict(
+            type='OpenSetLabelSmoothLoss', label_smooth_val=0.1, mode='original', unknown_target_zero=False),
+        cal_acc=False),
+    train_cfg=dict(_delete_=True),
+)
+
+bgr_mean = [123.675, 116.28, 103.53][::-1]
+bgr_std = [58.395, 57.12, 57.375][::-1]
+
+train_pipeline = [
+    dict(type='LoadImageFromFileFungi'),
+    dict(
+        type='RandomResizedCrop',
+        scale=384,
+        backend='pillow',
+        interpolation='bicubic'),
+    dict(type='RandomFlip', prob=0.5, direction='horizontal'),
+    dict(
+        type='RandAugment',
+        policies='timm_increasing',
+        num_policies=2,
+        total_level=10,
+        magnitude_level=9,
+        magnitude_std=0.5,
+        hparams=dict(
+            pad_val=[round(x) for x in bgr_mean], interpolation='bicubic')),
+    dict(
+        type='RandomErasing',
+        erase_prob=0.25,
+        mode='rand',
+        min_area_ratio=0.02,
+        max_area_ratio=1 / 3,
+        fill_color=bgr_mean,
+        fill_std=bgr_std),
+    dict(
+        type='PackMultiTaskInputs',
+        multi_task_fields=['gt_label']),
+]
+
+test_pipeline = [
+    dict(type='LoadImageFromFileFungi'),
+    dict(
+        type='ResizeEdge',
+        scale=438,
+        edge='short',
+        backend='pillow',
+        interpolation='bicubic'),
+    dict(type='CenterCrop', crop_size=384),
+    dict(
+        type='PackMultiTaskInputs',
+        multi_task_fields=['gt_label']),
+]
+
+train_dataloader = dict(
+    batch_size=16,
+    num_workers=10,
+    dataset=dict(
+        _delete_=True,
+        type='ConcatDataset',
+        datasets=[
+                dict(
+                    type='FungiMultitask',
+                    data_root='data/fungi2024/',
+                    ann_file='FungiCLEF2023_train_metadata_PRODUCTION.csv',
+                    data_prefix='DF20/',
+                    pipeline=train_pipeline),
+                dict(
+                    type='FungiMultitask',
+                    data_root='data/fungi2024/',
+                    ann_file='FungiCLEF2023_val_metadata_PRODUCTION.csv',
+                    data_prefix='DF21/',
+                    pipeline=train_pipeline,
+                    open_set=True),
+            ]))
+
+val_dataloader = dict(
+    batch_size=32,
+    num_workers=10,
+    dataset=dict(
+        type='FungiMultitask',
+        pipeline=test_pipeline))
+val_evaluator = dict(
+    _delete_=True,
+    type='MultiTasksMetric',
+    task_metrics=dict(
+        species=[dict(type='SingleLabelMetric', items=['precision', 'recall', 'f1-score'])],
+        genus=[dict(type='SingleLabelMetric', items=['precision', 'recall', 'f1-score'])],
+))
+
+train_cfg = dict(max_epochs=4)
+
+optim_wrapper = dict(type='AmpOptimWrapper', optimizer=dict(lr=5.e-4 * 64 / 512))
+
+# learning policy
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=0.01,
+        by_epoch=False,
+        end=2100),
+    # main learning rate scheduler
+    dict(type='CosineAnnealingLR', eta_min=0, by_epoch=False, begin=2100)
+]
+
+custom_imports = dict(imports=['mmpretrain_custom'], allow_failed_imports=False)
+
+load_from = 'work_dirs/swinv2_base_w24_b16x8-fp16_fungi+val2_res_384_genus-loss_no-unknown-target-zero/epoch_24.pth'

mmpretrain_custom/__init__.py

@@ -1 +1,2 @@
 from .datasets import *
+from .models import *

mmpretrain_custom/datasets/fungi.py

@@ -4,6 +4,7 @@ import pandas as pd
 import mmcv
 
 from mmpretrain.datasets.base_dataset import BaseDataset
+from mmpretrain.datasets.multi_task import MultiTaskDataset
 from mmpretrain.datasets.builder import DATASETS
 
 
@@ -726,19 +727,66 @@ class Fungi(BaseDataset):
     def load_data_list(self):
         table = pd.read_csv(self.ann_file)
 
-        def to_dict(cls_id, img_path):
-                # actual file endings are lower case for 300 px dataset
-            img_path = img_path.lower() if '300' in self.data_prefix else img_path
-            img_path = self.img_prefix + img_path
+        def to_dict(class_id, image_path, poisonous):
+            # actual file endings are lower case for 300 px dataset
+            image_path = image_path.lower() if '300' in self.data_prefix else image_path
+            image_path = self.img_prefix + image_path
             return {
-                'img_path': img_path,
-                'gt_label': np.array(cls_id, dtype=np.int64)
+                'img_path': image_path,
+                'gt_label': np.array(class_id, dtype=np.int64),
+                'poisonous': np.array(poisonous, dtype=np.int64)
+            }
+
+        # data_list = [to_dict(cls_id, img_path) for cls_id, img_path in zip(table['class_id'], table['image_path'])
+        #     if self.open_set or cls_id != -1]
+        data_list = [to_dict(**row) for row in table[['class_id', 'image_path', 'poisonous']].to_dict(orient='records')
+            if self.open_set or row['class_id'] != -1]
+        return data_list
+    
+@DATASETS.register_module()
+class FungiMultitask(MultiTaskDataset):
+    CLASSES = Fungi.CLASSES
+    def __init__(self, *args, ann_file, open_set=False, **kwargs):
+        self.open_set = open_set
+        super(FungiMultitask, self).__init__(*args, ann_file=ann_file, **kwargs)
+
+    def load_data_list(self, ann_file, metainfo_override=None):
+        # Set meta information.
+        metainfo = {}
+        assert isinstance(metainfo, dict), 'The `metainfo` field in the '\
+            f'annotation file should be a dict, but got {type(metainfo)}'
+        if metainfo_override is not None:
+            assert isinstance(metainfo_override, dict), 'The `metainfo` ' \
+                f'argument should be a dict, but got {type(metainfo_override)}'
+            metainfo.update(metainfo_override)
+        self._metainfo = self._get_meta_info(metainfo)
+
+        table = pd.read_csv(ann_file)
+
+        def to_dict(class_id, image_path, poisonous, genus):
+            # actual file endings are lower case for 300 px dataset
+            image_path = image_path.lower() if '300' in self.data_prefix else image_path
+            image_path = self.data_prefix + image_path
+            genus_idx = CLASSES_GENUS.index(genus)
+            assert genus_idx != -1
+            return {
+                'img_path': image_path,
+                'gt_label': {
+                    'species': class_id,
+                    'poisonous': poisonous,
+                    'genus': genus_idx,
+                }
             }
 
-        data_list = [to_dict(cls_id, img_path) for cls_id, img_path in zip(table['class_id'], table['image_path'])
-            if self.open_set or cls_id != -1]
+        # data_list = [to_dict(cls_id, img_path) for cls_id, img_path in zip(table['class_id'], table['image_path'])
+        #     if self.open_set or cls_id != -1]
+        data_list = [to_dict(**row) for row in table[['class_id', 'image_path', 'poisonous', 'genus']].to_dict(orient='records')
+            if self.open_set or row['class_id'] != -1]
         return data_list
     
+    def full_init(self):
+        pass
+    
 
 @DATASETS.register_module()
 class FungiTest(Fungi):
@@ -766,5 +814,35 @@ class FungiTest(Fungi):
                 'observation_id': obs_id
             }
 
-        data_list = [to_dict(img_path, obs_id) for img_path, obs_id in zip(table['image_path'], table['observation_id'])]
+        data_list = [to_dict(img_path, obs_id) for img_path, obs_id in zip(table['image_path'], table['observationID'])]
         return data_list
+
+CLASSES_GENUS = [
+    'Abortiporus', 'Acarospora', 'Achroomyces', 'Acrospermum', 'Adelphella', 'Agaricus', 'Agrocybe', 'Akenomyces', 'Albatrellus', 'Alboleptonia', 'Albugo', 'Aleuria', 'Aleurodiscus',
+                 'Allophylaria', 'Alnecium', 'Amandinea', 'Amanita', 'Amarenomyces', 'Amaropostia', 'Amphinema', 'Amphisphaerella', 'Ampulloclitocybe', 'Amyloporia', 'Amylostereum', 'Anaptychia',
+                 'Antella', 'Anthostomella', 'Anthracobia', 'Anthracoidea', 'Antrodia', 'Antrodiella', 'Apiculospora', 'Apiognomonia', 'Apioperdon', 'Arachnopeziza', 'Arcyria', 'Armillaria',
+                 'Arrhenia', 'Arthonia', 'Arthothelium', 'Arthrinium', 'Artomyces', 'Ascobolus', 'Ascochyta', 'Ascocorticium', 'Ascocoryne', 'Ascodichaena', 'Ascotremella', 'Asteromassaria',
+                 'Asterophora', 'Asterosporium', 'Astraeus', 'Athallia', 'Athelia', 'Athelopsis', 'Atheniella', 'Aurantiporus', 'Aureoboletus', 'Auricularia', 'Auriscalpium', 'Bacidia', 'Bactridium',
+                 'Badhamia', 'Baeomyces', 'Baeospora', 'Bartheletia', 'Beauveria', 'Belonium', 'Bertia', 'Bionectria', 'Biscogniauxia', 'Bispora', 'Bisporella', 'Bjerkandera', 'Blennothallia',
+                 'Blumeria', 'Boeremia', 'Bogbodia', 'Bolbitius', 'Boletus', 'Bombardia', 'Botryobasidium', 'Botryodiplodia', 'Botryotinia', 'Boubovia', 'Bovista', 'Bovistella', 'Brefeldia',
+                 'Bremia', 'Brevicellicium', 'Brunneoporus', 'Brunnipila', 'Bryoria', 'Bryoscyphus', 'Bryostigma', 'Buchwaldoboletus', 'Buellia', 'Buglossoporus', 'Bulbillomyces', 'Bulgaria',
+                 'Bulgariella', 'Butyriboletus', 'Byssocorticium', 'Byssomerulius', 'Byssonectria', 'Caeruleum', 'Calcipostia', 'Calicium', 'Callistosporium', 'Calloria', 'Caloboletus', 'Calocera',
+                 'Calocybe', 'Calogaya', 'Caloplaca', 'Caloscypha', 'Calosphaeria', 'Calospora', 'Calvatia', 'Calycellina', 'Calycina', 'Calyptella', 'Calyptospora', 'Camaropella',
+                 'Camarophyllopsis', 'Camarops', 'Camarosporidiella', 'Campanella', 'Candelaria', 'Candelariella', 'Cantharellula', 'Cantharellus', 'Capitotricha', 'Capronia', 'Catinella',
+                 'Cenangium', 'Cephalotrichum', 'Ceraceomyces', 'Ceratellopsis', 'Ceratiomyxa', 'Ceratostomella', 'Cerioporus', 'Ceriospora', 'Ceriporia', 'Ceriporiopsis', 'Cerrena', 'Cetraria',
+                 'Chaenotheca', 'Chaetosphaerella', 'Chalciporus', 'Chamaemyces', 'Cheilymenia', 'Chlorociboria', 'Chlorophyllum', 'Chloroscypha', 'Choiromyces', 'Chondrostereum', 'Chromocyphella',
+                 'Chromosera', 'Chroogomphus', 'Chrysomphalina', 'Chrysonectria', 'Chrysothrix', 'Ciboria', 'Cinereomyces', 'Circinaria', 'Cirrenalia', 'Cistella', 'Cladonia', 'Cladosporium',
+                 'Clathrus', 'Clavaria', 'Clavariadelphus', 'Claviceps', 'Clavicorona', 'Clavulina', 'Clavulinopsis', 'Climacocystis', 'Climacodon', 'Cliostomum', 'Clitocella', 'Clitocybe',
+                 'Clitocybula', 'Clitolyophyllum', 'Clitopaxillus', 'Clitopilus', 'Coenogonium', 'Coleophoma', 'Coleosporium', 'Coleroa', 'Collema', 'Colletotrichum', 'Collybia', 'Colpoma',
+                 'Coltricia', 'Comatricha', 'Comoclathris', 'Coniophora', 'Connopus', 'Conocybe', 'Contumyces', 'Coprinellus', 'Coprinopsis', 'Coprinus', 'Cordyceps', 'Coriolopsis', 'Corticifraga',
+                 'Corticium', 'Cortinarius', 'Coryneopsis', 'Coryneum', 'Cosmospora', 'Cotylidia', 'Craterellus', 'Craterium', 'Crepidotus', 'Cribraria', 'Crinipellis', 'Cristinia', 'Crocicreas',
+                 'Crucibulum', 'Cryptocoryneum', 'Cryptodiscus', 'Cryptosporella', 'Cucurbitaria', 'Cudonia', 'Cudoniella', 'Cumminsiella', 'Cuphophyllus', 'Cyanoboletus', 'Cyanosporus',
+                 'Cyathicula', 'Cyathus', 'Cyclaneusma', 'Cyclocybe', 'Cylindrobasidium', 'Cyphella', 'Cystoderma', 'Cystodermella', 'Cystolepiota', 'Cytospora', 'Dacrymyces', 'Dacryobolus',
+                 'Dactylaria', 'Daedalea', 'Daedaleopsis', 'Daldinia', 'Dasyscyphella', 'Deconica', 'Delicatula', 'Dendrocollybia', 'Dendrostilbella', 'Dendrothele', 'Dendryphiella', 'Dermea',
+                 'Dermoloma', 'Desmazierella', 'Dialonectria', 'Diapleella', 'Diaporthe', 'Diaporthopsis', 'Diatrype', 'Diatrypella', 'Dibaeis', 'Dichomitus', 'Dictydiaethalium', 'Diderma',
+                 'Didymella', 'Didymium', 'Didymocyrtis', 'Digitodochium', 'Diplocarpon', 'Diplodia', 'Diploicia', 'Diplomitoporus', 'Diplotomma', 'Discina', 'Disciotis', 'Discogloeum', 'Dissingia',
+                 'Ditiola', 'Dothidea', 'Dothiora', 'Dumontinia', 'Durella', 'Echinoderma', 'Echinosphaeria', 'Elaphomyces', 'Enchylium', 'Encoelia', 'Endoperplexa', 'Enerthenema', 'Engyodontium',
+                 'Enterographa', 'Entocybe', 'Entoloma', 'Entyloma', 'Epichloe', 'Epicoccum', 'Epithele', 'Erastia', 'Eriopezia', 'Erysiphe', 'Erythricium', 'Etheirodon', 'Euepixylon', 'Eutypa',
+                 'Eutypella', 'Evernia', 'Exidia', 'Exidiopsis', 'Exobasidium', 'Exophiala', 'Fayodia', 'Fellhanera', 'Fistulina', 'Flagelloscypha', 'Flammula', 'Flammulaster', 'Flammulina', 'Flavoparmelia', 'Flavoplaca', 'Flavoscypha', 'Fomes', 'Fomitiporia', 'Fomitopsis', 'Fuligo', 'Fusarium', 'Fuscoporia', 'Fuscopostia', 'Galerina', 'Gamundia', 'Ganoderma', 'Geastrum', 'Gelatoporia', 'Gemmina', 'Geoglossum', 'Geopora', 'Gerhardtia', 'Gibbera', 'Gibellula', 'Glaucomaria', 'Gliophorus', 'Globulicium', 'Gloeocystidiellum', 'Gloeophyllum', 'Gloeoporus', 'Gloiocephala', 'Gloiothele', 'Gloioxanthomyces', 'Glutinoglossum', 'Glyphium', 'Godronia', 'Golovinomyces', 'Gomphidius', 'Gomphus', 'Gonatophragmium', 'Granulobasidium', 'Graphis', 'Grifola', 'Guepinia', 'Guepiniopsis', 'Gymnopilus', 'Gymnopus', 'Gymnosporangium', 'Gyrodon', 'Gyromitra', 'Gyrophanopsis', 'Gyroporus', 'Haematomma', 'Hapalopilus', 'Hebeloma', 'Helicobasidium', 'Helminthosphaeria', 'Helminthosporium', 'Helvella', 'Hemileccinum', 'Hemileucoglossum', 'Hemimycena', 'Hemipholiota', 'Hemitrichia', 'Henningsomyces', 'Hericium', 'Heterobasidion', 'Heteromycophaga', 'Heteroradulum', 'Heterosphaeria', 'Hodophilus', 'Hohenbuehelia', 'Holwaya', 'Homophron', 'Hortiboletus', 'Humaria', 'Hyaloperonospora', 'Hyaloscypha', 'Hydnellum', 'Hydnoporia', 'Hydnum', 'Hydropunctaria', 'Hydropus', 'Hygrocybe', 'Hygrophoropsis', 'Hygrophorus', 'Hymenochaete', 'Hymenochaetopsis', 'Hymenopellis', 'Hymenoscyphus', 'Hyphoderma', 'Hyphodiscus', 'Hyphodontia', 'Hypholoma', 'Hypocenomyce', 'Hypochnicium', 'Hypocopra', 'Hypocreopsis', 'Hypoderma', 'Hypogymnia', 'Hypomyces', 'Hypotrachyna', 'Hypoxylon', 'Hysterangium', 'Hysterium', 'Hysterobrevium', 'Hysterostegiella', 'Illosporiopsis', 'Imleria', 'Imshaugia', 'Inermisia', 'Infundibulicybe', 'Inocutis', 'Inocybe', 'Inonotus', 'Inosperma', 'Intralichen', 'Iodophanus', 'Irpex', 'Isaria', 'Ischnoderma', 'Jackrogersella', 'Junghuhnia', 'Kavinia', 'Kretzschmaria', 'Kuehneola', 'Kuehneromyces', 'Kurtia', 'Laccaria', 'Lachnella', 'Lachnellula', 'Lachnum', 'Lacrymaria', 'Lactarius', 'Lactifluus', 'Laetiporus', 'Laetisaria', 'Lamproderma', 'Lamprospora', 'Lanzia', 'Lasallia', 'Lasiobelonium', 'Lasiosphaeria', 'Lasiosphaeris', 'Laxitextum', 'Lecanactis', 'Lecania', 'Lecanora', 'Leccinellum', 'Leccinum', 'Lecidea', 'Lecidella', 'Lemalis', 'Lentaria', 'Lentinellus', 'Lentinus', 'Lenzites', 'Leocarpus', 'Leotia', 'Lepiota', 'Lepista', 'Lepra', 'Lepraria', 'Leptoporus', 'Leptosphaeria', 'Leptosporomyces', 'Leptostroma', 'Leratiomyces', 'Leucoagaricus', 'Leucocoprinus', 'Leucocybe', 'Leucogyrophana', 'Leucopaxillus', 'Leucoscypha', 'Lichenochora', 'Lichenomphalia', 'Lichenopeltella', 'Limacella', 'Lindtneria', 'Lobaria', 'Lopadostoma', 'Lophiostoma', 'Lophiotrema', 'Lophodermium', 'Loreleia', 'Loweomyces', 'Luellia', 'Lycogala', 'Lycoperdon', 'Lylea', 'Lyomyces', 'Lyophyllum', 'Macbrideola', 'Macrocystidia', 'Macrolepiota', 'Macrotyphula', 'Marasmiellus', 'Marasmius', 'Massaria', 'Megacollybia', 'Melampsora', 'Melampsoridium', 'Melanconium', 'Melanelixia', 'Melanogaster', 'Melanohalea', 'Melanoleuca', 'Melanomma', 'Melanophyllum', 'Melanopsamma', 'Melanospora', 'Melastiza', 'Melogramma', 'Melomastia', 'Menispora', 'Mensularia', 'Meottomyces', 'Meripilus', 'Merismodes', 'Metatrichia', 'Micarea', 'Microbotryum', 'Microglossum', 'Microsphaeropsis', 'Microthyrium', 'Miladina', 'Milesina', 'Mitrophora', 'Mitrula', 'Mniaecia', 'Mollisia', 'Monilinia', 'Montagnula', 'Morchella', 'Moristroma', 'Mucidula', 'Mucilago', 'Mucronella', 'Mutinus', 'Mycena', 'Mycenastrum', 'Mycenella', 'Mycetinis', 'Mycoacia', 'Mycoaciella', 'Mycocalicium', 'Mycoglaena', 'Mycogone', 'Mycosphaerella', 'Myochromella', 'Myriolecis', 'Myxarium', 'Naetrocymbe', 'Naevala', 'Naohidemyces', 'Natantiella', 'Naucoria', 'Nectria', 'Nectriopsis', 'Nemania', 'Neoantrodia', 'Neoboletus', 'Neobulgaria', 'Neodasyscypha', 'Neoerysiphe', 'Neofavolus', 'Neofuscelia', 'Neolentinus', 'Neonectria', 'Neottiella', 'Nephromopsis', 'Nidularia', 'Nitschkia', 'Nodulisporium', 'Ochrolechia', 'Ochropsora', 'Octospora', 'Odoria', 'Oligoporus', 'Omphalina', 'Onygena', 'Opegrapha', 'Ophiobolus', 'Ophiognomonia', 'Orbilia', 'Ossicaulis', 'Otidea', 'Oudemansiella', 'Oxyporus', 'Pachyphiale', 'Pachyphlodes', 'Panaeolina', 'Panaeolus', 'Pandora', 'Panellus', 'Panus', 'Pappia', 'Paragymnopus', 'Paralepista', 'Parasola', 'Parmelia', 'Parmelina', 'Parmelinopsis', 'Parmeliopsis', 'Parmotrema', 'Patellaria', 'Paxillus', 'Peltigera', 'Penicillium', 'Peniophora', 'Peniophorella', 'Penttilamyces', 'Perenniporia', 'Perichaena', 'Peristemma', 'Peroneutypa', 'Peronospora', 'Perrotia', 'Pertusaria', 'Pestalotiopsis', 'Pezicula', 'Peziza', 'Phacidium', 'Phaeoclavulina', 'Phaeocollybia', 'Phaeohelotium', 'Phaeolepiota', 'Phaeolus', 'Phaeomarasmius', 'Phaeophyscia', 'Phaeosphaeria', 'Phaeotremella', 'Phallus', 'Phanerochaete', 'Phellinopsis', 'Phellinus', 'Phellodon', 'Phlebia', 'Phlebiella', 'Phlebiopsis', 'Phleogena', 'Phloeomana', 'Phlyctis', 'Pholiota', 'Pholiotina', 'Phomatospora', 'Phomopsis', 'Phragmidium', 'Phragmotrichum', 'Phycomyces', 'Phyllactinia', 'Phylloporia', 'Phylloporus', 'Phyllosticta', 'Phyllotopsis', 'Physalacria', 'Physalospora', 'Physarum', 'Physcia', 'Physconia', 'Physisporinus', 'Picipes', 'Pilaira', 'Pilobolus', 'Piloderma', 'Pisolithus', 'Pithya', 'Placynthiella', 'Plagiostoma', 'Plasmopara', 'Plasmoverna', 'Platismatia', 'Platygloea', 'Plectania', 'Plectosphaerella', 'Pleomassaria', 'Pleuroceras', 'Pleurocybella', 'Pleurosticta', 'Pleurotus', 'Plicaturopsis', 'Pluteus', 'Podosordaria', 'Podosphaera', 'Podospora', 'Polycauliona', 'Polycephalomyces', 'Polycoccum', 'Polydesmia', 'Polyozosia', 'Polyporus', 'Polysporina', 'Polythrincium', 'Porodaedalea', 'Poronia', 'Porostereum', 'Porotheleum', 'Porphyrellus', 'Porpidia', 'Porpolomopsis', 'Postia', 'Proliferodiscus', 'Propolis', 'Prosthecium', 'Protoblastenia', 'Protocrea', 'Protomyces', 'Protoparmeliopsis', 'Protostropharia', 'Protounguicularia', 'Psathyrella', 'Pseudevernia', 'Pseudobaeospora', 'Pseudoboletus', 'Pseudoclitocybe', 'Pseudoclitopilus', 'Pseudocraterellus', 'Pseudohydnum', 'Pseudoinonotus', 'Pseudolaccaria', 'Pseudolachnea', 'Pseudonectria', 'Pseudopeziza', 'Pseudophacidium', 'Pseudoplectania', 'Pseudosagedia', 'Pseudoschismatomma', 'Pseudosperma', 'Pseudotricholoma', 'Psilachnum', 'Psilocybe', 'Psilolechia', 'Pterula', 'Pterulicium', 'Puccinia', 'Pucciniastrum', 'Pulvinula', 'Punctelia', 'Pustula', 'Pycnoporellus', 'Pycnoporus', 'Pyrenopeziza', 'Pyrenula', 'Pyronema', 'Radulodon', 'Radulomyces', 'Ramalina', 'Ramaria', 'Ramariopsis', 'Ramularia', 'Rebentischia', 'Resinicium', 'Resiniporus', 'Resinomycena', 'Resupinatus', 'Reticularia', 'Rhizina', 'Rhizocarpon', 'Rhizochaete', 'Rhizoctonia', 'Rhizomarasmius', 'Rhizopogon', 'Rhodocollybia', 'Rhopographus', 'Rhytisma', 'Rickenella', 'Riessia', 'Rigidoporus', 'Rimbachia', 'Rinodina', 'Ripartites', 'Roridomyces', 'Rosellinia', 'Roseodiscus', 'Rubroboletus', 'Rugosomyces', 'Russula', 'Rutola', 'Rutstroemia', 'Ruzenia', 'Sabuloglossum', 'Saccobolus', 'Sagaranella', 'Sarcodon', 'Sarcogyne', 'Sarcopodium', 'Sarcoscypha', 'Sarcosphaera', 'Sarea', 'Sawadaea', 'Schizophyllum', 'Schizopora', 'Schizothecium', 'Sclerencoelia', 'Scleroderma', 'Sclerotinia', 'Scopuloides', 'Scutellinia', 'Scytinium', 'Scytinostroma', 'Sebacina', 'Seifertia', 'Seimatosporium', 'Sepedonium', 'Septoria', 'Serpula', 'Setiferotheca', 'Sidera', 'Simocybe', 'Sirococcus', 'Sistotrema', 'Sistotremastrum', 'Skeletocutis', 'Sordaria', 'Sparassis', 'Spathularia', 'Spermosporina', 'Sphacelotheca', 'Sphaeridium', 'Sphaerobolus', 'Sphaerographium', 'Sphaeropsis', 'Sphaerulina', 'Sphagnurus', 'Spinellus', 'Splanchnonema', 'Sporothrix', 'Stagonospora', 'Steccherinum', 'Stemonitis', 'Stemonitopsis', 'Stemphylium', 'Stereocaulon', 'Stereopsis', 'Stereum', 'Stictis', 'Stilbella', 'Straminella', 'Strangospora', 'Strobilomyces', 'Strobilurus', 'Stropharia', 'Subulicystidium', 'Suillellus', 'Suillus', 'Symphytocarpus', 'Synaptospora', 'Synchytrium', 'Syzygites', 'Syzygospora', 'Szczepkamyces', 'Tapesia', 'Taphrina', 'Tapinella', 'Tarzetta', 'Tephrocybe', 'Tephromela', 'Terana', 'Thecaphora', 'Thecotheus', 'Thecotubifera', 'Thelephora', 'Thelotrema', 'Thyridaria', 'Thyronectria', 'Tolypocladium', 'Tomentella', 'Torula', 'Trachyspora', 'Trametes', 'Tranzschelia', 'Trapelia', 'Trapeliopsis', 'Trechispora', 'Tremella', 'Tremellodendropsis', 'Trichaptum', 'Trichia', 'Trichioides', 'Trichobelonium', 'Trichobolus', 'Trichoderma', 'Trichoglossum', 'Tricholoma', 'Tricholomella', 'Tricholomopsis', 'Trichopeziza', 'Trichopezizella', 'Trichophaea', 'Trichosphaerella', 'Trimmatostroma', 'Triphragmium', 'Trochila', 'Trullula', 'Tubaria', 'Tuber', 'Tubeufia', 'Tubifera', 'Tubulicrinis', 'Tuckermannopsis', 'Tulasnella', 'Tulostoma', 'Tylopilus', 'Tylospora', 'Tympanis', 'Typhula', 'Tyromyces', 'Umbilicaria', 'Unguiculariopsis', 'Urocystis', 'Uromyces', 'Usnea', 'Ustilago', 'Valsa', 'Variospora', 'Velutarina', 'Venturia', 'Venturiocistella', 'Verpa', 'Verrucaria', 'Verrucoplaca', 'Vialaea', 'Vitreoporus', 'Volutella', 'Volvariella', 'Volvopluteus', 'Vouauxiella', 'Vuilleminia', 'Vulpicida', 'Wallrothiella', 'Woldmaria', 'Xanthocarpia', 'Xanthoparmelia', 'Xanthoporia', 'Xanthoria', 'Xanthoriicola', 'Xenasmatella', 'Xenotypa', 'Xerocomellus', 'Xeromphalina', 'Xerula', 'Xylaria', 'Xylodon', 'Xylohypha'
+                 ]
+

mmpretrain_custom/models/__init__.py

@@ -0,0 +1,2 @@
+from .heads import *
+from .losses import *

mmpretrain_custom/models/classifiers/metadata.py

@@ -0,0 +1,45 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from ..builder import CLASSIFIERS
+from ..heads import MultiLabelClsHead
+from .image import ImageClassifier
+
+
+@CLASSIFIERS.register_module()
+class MetadataClassifier(ImageClassifier):
+
+    def forward_train(self, img, gt_label, img_metas, **kwargs):
+        """Forward computation during training.
+
+        Args:
+            img (Tensor): of shape (N, C, H, W) encoding input images.
+                Typically these should be mean centered and std scaled.
+            gt_label (Tensor): It should be of shape (N, 1) encoding the
+                ground-truth label of input images for single label task. It
+                shoulf be of shape (N, C) encoding the ground-truth label
+                of input images for multi-labels task.
+        Returns:
+            dict[str, Tensor]: a dictionary of loss components
+        """
+        if self.augments is not None:
+            img, gt_label = self.augments(img, gt_label)
+
+        x = self.extract_feat(img)
+
+        losses = dict()
+        loss = self.head.forward_train(x, gt_label, img_metas)
+
+        losses.update(loss)
+
+        return losses
+
+    def simple_test(self, img, img_metas=None, **kwargs):
+        """Test without augmentation."""
+        x = self.extract_feat(img)
+
+        if isinstance(self.head, MultiLabelClsHead):
+            assert 'softmax' not in kwargs, (
+                'Please use `sigmoid` instead of `softmax` '
+                'in multi-label tasks.')
+        res = self.head.simple_test(x, img_metas, **kwargs)
+
+        return res

mmpretrain_custom/models/heads/__init__.py

@@ -0,0 +1 @@
+from .metadata_head import MetadataHead

mmpretrain_custom/models/heads/metadata_head.py

@@ -0,0 +1,322 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import math
+import os.path as osp
+from typing import List, Optional, Tuple
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+import numpy as np
+
+import pandas as pd
+
+from mmpretrain.registry import MODELS
+from mmpretrain.models import LinearClsHead
+from mmpretrain.structures import DataSample
+
+
+@MODELS.register_module()
+class MetadataHead(LinearClsHead):
+    def __init__(self,
+                 in_channels,
+                 init_cfg=dict(type='Normal', layer='Linear', std=0.01),
+                 data_paths=[],
+                #  location=False,
+                #  location_only=False,
+                #  meta_norm=False,
+                #  meta_pre_norm_only=False,
+                #  meta_no_pre_norm=False,
+                #  meta_dropout=False,
+                 **kwargs):
+        
+        super(MetadataHead, self).__init__(in_channels=in_channels + 64, init_cfg=init_cfg, **kwargs)
+
+        # self.location = location
+        # self.location_only = location_only
+        # if self.location_only:
+        #     assert self.location
+
+        COUNTRY_CODES_TRAIN = ['AL', 'AT', 'AU', 'CA', 'CR', 'CZ', 'DE', 'DK', 'ES', 'FI', 'FO', 'FR', 'GA', 'GB', 'GL', 'GR', 'HR', 'HU', 'IS', 'IT', 'JP', 'NL', 'NO', 'NP', 'PL', 'PT', 'RU', 'SE', 'SJ', 'US']
+        COUNTRY_CODES_TRAINVAL = ['AL', 'AT', 'AU', 'BE', 'CA', 'CH', 'CR', 'CZ', 'DE', 'DK', 'ES', 'FI', 'FO', 'FR', 'GA', 'GB', 'GL', 'GR', 'HR', 'HU', 'IS', 'IT', 'JP', 'NL', 'NO', 'NP', 'PL', 'PT', 'RU', 'SE', 'SJ', 'US']
+
+        SUBSTRATES_TRAIN = ['bark of living trees', 'building stone (e.g. bricks)', 'calcareous stone', 'catkins', 'cones', 'dead stems of herbs, grass etc', 'dead wood (including bark)', 'faeces', 'fire spot', 'fruits', 'fungi', 'insects', 'leaf or needle litter', 'lichens', 'liverworts', 'living flowers', 'living leaves', 'living stems of herbs, grass etc', 'mosses', 'mycetozoans', 'other substrate', 'peat mosses', 'remains of vertebrates (e.g. feathers and fur)', 'siliceous stone', 'soil', 'stems of herbs, grass etc', 'stone', 'wood and roots of living trees', 'wood chips or mulch']
+        SUBSTRATES_TRAINVAL = ['bark of living trees', 'building stone (e.g. bricks)', 'calcareous stone', 'catkins', 'cones', 'dead stems of herbs, grass etc', 'dead wood (including bark)', 'faeces', 'fire spot', 'fruits', 'fungi', 'insects', 'leaf or needle litter', 'lichens', 'liverworts', 'living flowers', 'living leaves', 'living stems of herbs, grass etc', 'mosses', 'mycetozoans', 'other substrate', 'peat mosses', 'remains of vertebrates (e.g. feathers and fur)', 'siliceous stone', 'soil', 'spiders', 'stems of herbs, grass etc', 'stone', 'wood and roots of living trees', 'wood chips or mulch']
+
+        HABITAT_TRAIN = ['Acidic oak woodland', 'Bog woodland', 'Deciduous woodland', 'Forest bog', 'Mixed woodland (with coniferous and deciduous trees)', 'Thorny scrubland', 'Unmanaged coniferous woodland', 'Unmanaged deciduous woodland', 'Willow scrubland', 'bog', 'coniferous woodland/plantation', 'ditch', 'dune', 'fallow field', 'fertilized field in rotation', 'garden', 'gravel or clay pit', 'heath', 'hedgerow', 'improved grassland', 'lawn', 'masonry', 'meadow', 'natural grassland', 'other habitat', 'park/churchyard', 'roadside', 'rock', 'roof', 'salt meadow', 'wooded meadow, grazing forest']
+
+        metadata = [pd.read_csv(path) for path in data_paths]
+        self.meta_features = dict()
+        for m in metadata:
+            for idx, row in m.iterrows():
+                month = torch.tensor([math.sin(2 * math.pi * row['month'] / 12), math.cos(2 * math.pi * row['month'] / 12)], dtype=torch.float32)
+                day = torch.tensor([math.sin(2 * math.pi * row['day'] / 31), math.cos(2 * math.pi * row['day'] / 31)], dtype=torch.float32)
+                
+                country_code = torch.zeros(size=(len(COUNTRY_CODES_TRAIN),), dtype=torch.float32)
+                try:
+                    country_code_index = COUNTRY_CODES_TRAIN.index(row['countryCode'])
+                    country_code = F.one_hot(torch.tensor(country_code_index), num_classes=country_code.shape[0])
+                except ValueError:
+                    pass
+
+                substrates = torch.zeros(size=(len(SUBSTRATES_TRAIN),), dtype=torch.float32)
+                try:
+                    substrates_index = SUBSTRATES_TRAIN.index(row['Substrate'])
+                    substrates = F.one_hot(torch.tensor(substrates_index), num_classes=substrates.shape[0])
+                except ValueError:
+                    pass
+                
+                habitat = torch.zeros(size=(len(HABITAT_TRAIN),), dtype=torch.float32)
+                try:
+                    habitat_index = HABITAT_TRAIN.index(row['Habitat'])
+                    habitat = F.one_hot(torch.tensor(habitat_index), num_classes=habitat.shape[0])
+                except ValueError:
+                    pass
+
+                meta_features = torch.cat([month, day, country_code, substrates, habitat])
+                
+                self.meta_features[row['image_path']] = meta_features
+
+        self.meta_in_channels = len(meta_features)
+        self.meta_fc1 = nn.Linear(self.meta_in_channels, 64)
+        self.meta_fc2 = nn.Linear(64, 64)
+
+        self.meta_norm1 = nn.LayerNorm((64,))
+        self.meta_norm2 = nn.LayerNorm((64,))
+
+        # self.meta_norm = meta_norm
+        # self.meta_pre_norm_only = meta_pre_norm_only
+        # self.meta_no_pre_norm = meta_no_pre_norm
+        # if self.meta_pre_norm_only or self.meta_no_pre_norm:
+        #     assert self.meta_norm
+        # assert not (self.meta_pre_norm_only and self.meta_no_pre_norm)
+        # if self.meta_norm:
+        #     if not self.meta_no_pre_norm:
+        #         self.meta_norm1 = nn.LayerNorm((self.meta_in_channels,))
+        #     if not self.meta_pre_norm_only:
+        #         self.meta_norm2 = nn.LayerNorm((256,))
+        #         self.meta_norm3 = nn.LayerNorm((256,))
+        #         self.meta_norm4 = nn.LayerNorm((256,))
+
+        # self.meta_dropout = meta_dropout
+        # if self.meta_dropout:
+        #     self.meta_dropout1 = nn.Dropout(0.1)
+        #     self.meta_dropout2 = nn.Dropout(0.1)
+        #     self.meta_dropout3 = nn.Dropout(0.1)
+        
+        # self.fc = nn.Linear(self.in_channels + 256, self.num_classes)
+
+        # train_meta = pd.read_csv('/net/fulu/storage/deeplearning/users/wolfst/fungiclef/2022/DF20-train_metadata.csv')
+        # val_meta = pd.read_csv('/net/fulu/storage/deeplearning/users/wolfst/fungiclef/2022/DF20-val_metadata.csv')
+        # test_meta = pd.read_csv('/net/fulu/storage/deeplearning/users/wolfst/fungiclef/2022/FungiCLEF2022_test_metadata.csv')
+
+        # substrate_classes = set(train_meta['Substrate']).union(set(val_meta['Substrate'])).union(set(test_meta['Substrate']))
+        # # remove nan
+        # substrate_classes = {x for x in substrate_classes if pd.notna(x)}
+        # substrate_mapping = dict([(x, i) for i, x in list(enumerate(substrate_classes))])
+        # def substrate_to_one_hot(substrate):
+        #     one_hot = np.zeros((len(substrate_classes), ))
+        #     if pd.notna(substrate):
+        #         one_hot[substrate_mapping[substrate]] = 1.
+        #     return one_hot
+
+        # X_train_substrate = np.array([substrate_to_one_hot(x) for x in train_meta['Substrate']])
+        # X_val_substrate = np.array([substrate_to_one_hot(x) for x in val_meta['Substrate']])
+        # X_test_substrate = np.array([substrate_to_one_hot(x) for x in test_meta['Substrate']])
+
+        # habitat_classes = set(train_meta['Habitat']).union(set(val_meta['Habitat'])).union(set(test_meta['Habitat']))
+        # # remove nan
+        # habitat_classes = {x for x in habitat_classes if pd.notna(x)}
+        # habitat_mapping = dict([(x, i) for i, x in list(enumerate(habitat_classes))])
+        # def habitat_to_one_hot(habitat):
+        #     one_hot = np.zeros((len(habitat_classes), ))
+        #     if pd.notna(habitat):
+        #         one_hot[habitat_mapping[habitat]] = 1.
+        #     return one_hot
+
+        # X_train_habitat = np.array([habitat_to_one_hot(x) for x in train_meta['Habitat']])
+        # X_val_habitat = np.array([habitat_to_one_hot(x) for x in val_meta['Habitat']])
+        # X_test_habitat = np.array([habitat_to_one_hot(x) for x in test_meta['Habitat']])
+
+        # X_train_month = np.expand_dims(train_meta['month'], axis=1)
+        # X_train_month = X_train_month.copy()
+        # X_train_month[~pd.notna(X_train_month)] = 0.
+        # X_val_month = np.expand_dims(val_meta['month'], axis=1)
+        # X_val_month = X_val_month.copy()
+        # X_val_month[~pd.notna(X_val_month)] = 0.
+        # X_test_month = np.expand_dims(test_meta['month'], axis=1)
+        # X_test_month = X_test_month.copy()
+        # X_test_month[~pd.notna(X_test_month)] = 0.
+
+        # if self.location:
+        #     X_train_location = np.array(list(zip(train_meta['Latitude'], train_meta['Longitude'])))
+        #     X_train_location = X_train_location.copy()
+        #     X_train_location[~pd.notna(X_train_location[:, 0]) | ~pd.notna(X_train_location[:, 1])] = 0.
+        #     X_val_location = np.array(list(zip(val_meta['Latitude'], val_meta['Longitude'])))
+        #     X_val_location = X_val_location.copy()
+        #     X_val_location[~pd.notna(X_val_location[:, 0]) | ~pd.notna(X_val_location[:, 1])] = 0.
+            
+        #     location_mapping = pd.read_csv('test_location_mapping.csv')
+        #     location_mapping = {town: (lat, lon) for town, lat, lon in
+        #         zip(location_mapping['Location_lvl1'], location_mapping['Latitude'], location_mapping['Longitude'])}
+        #     X_test_location = np.array([location_mapping[town] if isinstance(town, str) else (0., 0.) for town in test_meta['Location_lvl1']])
+            
+        #     if self.location_only:
+        #         X_train = X_train_location
+        #         X_val = X_val_location
+        #         X_test = X_test_location
+        #     else:
+        #         X_train = np.concatenate([X_train_substrate, X_train_habitat, X_train_month, X_train_location], axis=1)
+        #         X_val = np.concatenate([X_val_substrate, X_val_habitat, X_val_month, X_val_location], axis=1)
+        #         X_test = np.concatenate([X_test_substrate, X_test_habitat, X_test_month, X_test_location], axis=1)
+        # else:
+        #     X_train = np.concatenate([X_train_substrate, X_train_habitat, X_train_month], axis=1)
+        #     X_val = np.concatenate([X_val_substrate, X_val_habitat, X_val_month], axis=1)
+        #     X_test = np.concatenate([X_test_substrate, X_test_habitat, X_test_month], axis=1)
+
+        # self.train_meta_feats = dict([(img_path.lower(), torch.tensor(feats)) for img_path, feats in zip(train_meta['image_path'], X_train)])
+        # #self.train_meta_feats = dict([(img_path, torch.tensor(feats)) for img_path, feats in zip(train_meta['image_path'], X_train)])
+        # self.val_meta_feats = dict([(img_path.lower(), torch.tensor(feats)) for img_path, feats in zip(val_meta['image_path'], X_val)])
+        # #self.val_meta_feats = dict([(img_path, torch.tensor(feats)) for img_path, feats in zip(val_meta['image_path'], X_val)])
+        # self.test_meta_feats = dict([(filename, torch.tensor(feats)) for filename, feats in zip(test_meta['filename'], X_test)])
+
+
+    def forward(self, feats: Tuple[torch.Tensor], img_paths: str) -> torch.Tensor:
+        """The forward process."""
+        pre_logits = self.pre_logits(feats)
+        # The final classification head.
+        meta_feats = torch.stack([self.meta_features[img_path].to(feats[0].device) for img_path in img_paths], dim=0)
+        meta_feats = self.meta_norm1(F.relu(self.meta_fc1(meta_feats)))
+        meta_feats = self.meta_norm2(F.relu(self.meta_fc2(meta_feats))) + meta_feats
+        pre_logits = torch.cat([pre_logits, meta_feats], dim=1)
+        cls_score = self.fc(pre_logits)
+        return cls_score
+    
+    def loss(self, feats: Tuple[torch.Tensor], data_samples: List[DataSample],
+             **kwargs) -> dict:
+        """Calculate losses from the classification score.
+
+        Args:
+            feats (tuple[Tensor]): The features extracted from the backbone.
+                Multiple stage inputs are acceptable but only the last stage
+                will be used to classify. The shape of every item should be
+                ``(num_samples, num_classes)``.
+            data_samples (List[DataSample]): The annotation data of
+                every samples.
+            **kwargs: Other keyword arguments to forward the loss module.
+
+        Returns:
+            dict[str, Tensor]: a dictionary of loss components
+        """
+        # The part can be traced by torch.fx
+        cls_score = self(feats, [osp.basename(x.img_path) for x in data_samples])
+
+        # The part can not be traced by torch.fx
+        losses = self._get_loss(cls_score, data_samples, **kwargs)
+        return losses
+    
+    def predict(
+        self,
+        feats: Tuple[torch.Tensor],
+        data_samples: Optional[List[Optional[DataSample]]] = None,
+        img_paths: Optional[List[str]] = None
+    ) -> List[DataSample]:
+        """Inference without augmentation.
+
+        Args:
+            feats (tuple[Tensor]): The features extracted from the backbone.
+                Multiple stage inputs are acceptable but only the last stage
+                will be used to classify. The shape of every item should be
+                ``(num_samples, num_classes)``.
+            data_samples (List[DataSample | None], optional): The annotation
+                data of every samples. If not None, set ``pred_label`` of
+                the input data samples. Defaults to None.
+
+        Returns:
+            List[DataSample]: A list of data samples which contains the
+            predicted results.
+        """
+        # The part can be traced by torch.fx
+        if img_paths is None:
+            img_paths = [osp.basename(x.img_path) for x in data_samples]
+        cls_score = self(feats, img_paths)
+
+        # The part can not be traced by torch.fx
+        predictions = self._get_predictions(cls_score, data_samples)
+        return predictions
+
+        
+    def simple_test(self, x, img_metas, softmax=True, post_process=True):
+        """Inference without augmentation.
+
+        Args:
+            x (tuple[Tensor]): The input features.
+                Multi-stage inputs are acceptable but only the last stage will
+                be used to classify. The shape of every item should be
+                ``(num_samples, in_channels)``.
+            softmax (bool): Whether to softmax the classification score.
+            post_process (bool): Whether to do post processing the
+                inference results. It will convert the output to a list.
+
+        Returns:
+            Tensor | list: The inference results.
+
+                - If no post processing, the output is a tensor with shape
+                  ``(num_samples, num_classes)``.
+                - If post processing, the output is a multi-dimentional list of
+                  float and the dimensions are ``(num_samples, num_classes)``.
+        """
+        x = self.pre_logits(x)
+
+        #x_meta = [self.val_meta_feats[osp.basename(img_meta['filename'])] for img_meta in img_metas]
+        x_meta = [self.test_meta_feats[osp.basename(img_meta['filename'])] for img_meta in img_metas]
+        x_meta = torch.stack(x_meta, dim=0).to(x.device, x.dtype)
+        x_meta = F.relu(self.meta_fc1(x_meta))
+        x_meta_identity = x_meta
+        x_meta = F.relu(self.meta_fc2(x_meta))
+        x_meta = F.relu(self.meta_fc3(x_meta)) + x_meta_identity
+
+        x = torch.cat([x, x_meta], dim=1)
+
+        cls_score = self.fc(x)
+
+        if softmax:
+            pred = (
+                F.softmax(cls_score, dim=1) if cls_score is not None else None)
+        else:
+            pred = cls_score
+
+        if post_process:
+            return self.post_process(pred)
+        else:
+            return pred
+
+    def forward_train(self, x, gt_label, img_metas, **kwargs):
+        x = self.pre_logits(x)
+
+        x_meta = [self.train_meta_feats[osp.basename(img_meta['filename'])] for img_meta in img_metas]
+        x_meta = torch.stack(x_meta, dim=0).to(x.device, x.dtype)
+        if self.meta_norm and not self.meta_no_pre_norm:
+            x_meta = self.meta_norm1(x_meta)
+        x_meta = F.relu(self.meta_fc1(x_meta))
+        if self.meta_norm and not self.meta_pre_norm_only:
+            x_meta = self.meta_norm2(x_meta)
+        if self.meta_dropout:
+            x_meta = self.meta_dropout1(x_meta)
+        x_meta_identity = x_meta
+        x_meta = F.relu(self.meta_fc2(x_meta))
+        if self.meta_norm and not self.meta_pre_norm_only:
+            x_meta = self.meta_norm3(x_meta)
+        if self.meta_dropout:
+            x_meta = self.meta_dropout2(x_meta)
+        x_meta = F.relu(self.meta_fc3(x_meta)) + x_meta_identity
+        if self.meta_norm and not self.meta_pre_norm_only:
+            x_meta = self.meta_norm4(x_meta)
+        if self.meta_dropout:
+            x_meta = self.meta_dropout3(x_meta)
+
+        x = torch.cat([x, x_meta], dim=1)
+
+        cls_score = self.fc(x)
+        losses = self.loss(cls_score, gt_label, **kwargs)
+        return losses

mmpretrain_custom/models/losses/__init__.py

@@ -0,0 +1 @@
+from .open_set_label_smooth_loss import OpenSetLabelSmoothLoss

mmpretrain_custom/models/losses/open_set_label_smooth_loss.py

@@ -0,0 +1,22 @@
+import torch
+
+from mmpretrain.models import LabelSmoothLoss
+from mmpretrain.registry import MODELS
+
+
+@MODELS.register_module()
+class OpenSetLabelSmoothLoss(LabelSmoothLoss):
+
+    def __init__(self, unknown_target_zero=True, **kwargs):
+        super().__init__(**kwargs)
+        self.unknown_target_zero = unknown_target_zero
+
+    def generate_one_hot_like_label(self, label):
+        """This function takes one-hot or index label vectors and computes one-
+        hot like label vectors (float)"""
+        one_hot_labels = torch.zeros((label.shape[0], self.num_classes), dtype=torch.float, device=label.device)
+        if not self.unknown_target_zero:
+            one_hot_labels[:] = 1. / self.num_classes
+        closed_set_labels = label != -1
+        one_hot_labels[closed_set_labels] = super().generate_one_hot_like_label(label[closed_set_labels])
+        return one_hot_labels

script.py

@@ -1,43 +1,109 @@
 import pandas as pd
 import numpy as np
 import os
-import subprocess
 import sys
 from tqdm import tqdm
 import timm
 import torchvision.transforms as T
 from PIL import Image
 import torch
+from multiprocessing import Pool
 
+from mmpretrain.apis import ImageClassificationInferencer, FeatureExtractor
 
-# custom script arguments
-CONFIG_PATH = 'models/swinv2_base_w24_b16x4-fp16_fungi+val_res_384_cb_epochs_6.py'
-CHECKPOINT_PATH = "models/swinv2_base_w24_b16x4-fp16_fungi+val_res_384_cb_epochs_6_epoch_6_20240514-de00365e.pth"
-SCORE_THRESHOLD = 0.0
-
-def run_inference(input_csv, output_csv, data_root_path):
-    """Load model and dataloader and run inference."""
-
-    if not data_root_path.endswith('/'):
-        data_root_path += '/'
-    data_cfg_opts = [
-        f'test_dataloader.dataset.data_root=',
-        f'test_dataloader.dataset.ann_file={input_csv}',
-        f'test_dataloader.dataset.data_prefix={data_root_path}']
-
-    inference = subprocess.Popen([
-        'python', '-m',
-        'tools.test_generate_result_pre-consensus',
-        CONFIG_PATH, CHECKPOINT_PATH,
-        output_csv,
-        '--threshold', str(SCORE_THRESHOLD),
-        '--no-scores',
-        '--cfg-options'] + data_cfg_opts)
-    return_code = inference.wait()
-    if return_code != 0:
-        print(f'Inference crashed with exit code {return_code}')
-        sys.exit(return_code)
-    print(f'Written {output_csv}')
+import mmpretrain.utils.progress as progress
+progress.disable_progress_bar = True
+
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
+
+
+def load_image(path : str, images_root_path="/tmp/data/private_testset"):
+    return np.array(Image.open(os.path.join(images_root_path, path)))[:, :, ::-1]
+
+def rerank_poison(posison_status_list : pd.DataFrame, pred_scores : np.array) -> tuple[int, float]:
+    class_id = np.argmax(pred_scores)
+    class_score = np.max(pred_scores)
+
+    poisonous = posison_status_list.copy()
+    poisonous['score'] = pred_scores
+    poisonous.sort_values(by=['score'], ascending=False, inplace=True)
+    first_poisonous = poisonous[poisonous['poisonous'] == 1].iloc[0]
+
+    if 13 * first_poisonous['score'] > class_score:
+        class_id = first_poisonous['class_id']
+        class_score = first_poisonous['score']
+
+    return class_id, class_score
+
+
+def make_submission(test_metadata, model_path, model_name, output_csv_path="./submission.csv", images_root_path="/tmp/data/private_testset"):
+    """Make submission with given """
+
+    #inferencer = ImageClassificationInferencer(model=model_name, pretrained=model_path, device="cuda:0")
+    feature_extractor = FeatureExtractor(model=model_name, pretrained=model_path, device="cuda:0")
+
+    predictions = []
+    prediction_scores = []
+    prediction_scores_dict = {}
+    prediction_feats_dict = {}
+    obs_imgs_dict = {}
+
+    BATCH_SIZE = 4
+    p = Pool(BATCH_SIZE)
+    # image_paths_next_batch = test_metadata['image_path'][0:BACTH_SIZE]
+    # next_batch = p.map_async(load_image, image_paths_next_batch)
+    for i in tqdm(range(int(np.ceil(test_metadata.shape[0] / BATCH_SIZE)))):
+        # batch_imgs = next_batch.get()
+        # image_paths_next_batch = test_metadata['image_path'][(i+1) * BACTH_SIZE:(i+2) * BATCH_SIZE]
+        # next_batch = p.map_async(load_image, image_paths_next_batch)
+        img_paths_batch = test_metadata['image_path'][(i) * BATCH_SIZE:(i+1) * BATCH_SIZE]
+        batch_imgs = p.map(load_image, img_paths_batch)
+        #batch_imgs = [np.array(Image.open(os.path.join(images_root_path, x)))[:, :, ::-1] for x in test_metadata['image_path'][(i) * BATCH_SIZE:(i+1) * BATCH_SIZE]]
+        #results = inferencer(batch_imgs, batch_size=BATCH_SIZE)
+        feats = feature_extractor(batch_imgs, batch_size=BATCH_SIZE)
+        feats = (torch.stack([x[0] for x in feats], dim=0),)
+        results = feature_extractor.model.head.task_heads['species'].predict(feats, img_paths=img_paths_batch)
+        for res, f, obs_id, img_path in zip(results, feats[0], test_metadata['observation_id'][(i) * BATCH_SIZE:(i+1) * BATCH_SIZE], img_paths_batch):
+            #pred_scores = res.species.pred_score.detach().cpu().numpy()
+            pred_scores = res.pred_score.detach().cpu().numpy()
+            #pred_scores = res['pred_scores']
+            predictions.append(np.argmax(pred_scores))
+            prediction_scores.append(pred_scores)
+            prediction_scores_dict.setdefault(obs_id, []).append(pred_scores)
+            prediction_feats_dict.setdefault(obs_id, []).append(f)
+            obs_imgs_dict[obs_id] = img_path
+
+    print('finished inference')
+
+    test_metadata["class_id"] = predictions
+    test_metadata["max_score"] = prediction_scores
+
+    poison_status_list = pd.read_csv('poison_status_list.csv')
+    poison_status_list = poison_status_list.sort_values(by=['class_id'])
+
+    poison_classes = set(poison_status_list[poison_status_list['poisonous'] == 1]['class_id'])
+
+    for obs_id, pred_feats in tqdm(prediction_feats_dict.items()):
+        #fusion_scores = np.prod(np.array(pred_scores), axis=0)
+        #fusion_scores = np.mean(np.array(pred_scores), axis=0)
+        #fusion_scores = np.max(np.array(pred_scores), axis=0)
+        fusion_feats = torch.mean(torch.stack(pred_feats, dim=0), dim=0, keepdim=True)
+        results = feature_extractor.model.head.task_heads['species'].predict((fusion_feats,), img_paths=[obs_imgs_dict[obs_id]])
+        fusion_scores = results[0].pred_score.detach().cpu().numpy()
+        class_score = np.max(fusion_scores)
+        class_id = np.argmax(fusion_scores)
+        class_id, class_score = rerank_poison(poison_status_list, fusion_scores)
+        entropy = -np.sum(fusion_scores * np.log(fusion_scores))
+        if entropy > 7 or (class_id not in poison_classes and entropy > 2.5):
+            class_id = -1
+        #if class_score < 0.1:
+        #    class_id = -1
+        test_metadata.loc[test_metadata["observation_id"] == obs_id, "class_id"] = class_id
+        test_metadata.loc[test_metadata["observation_id"] == obs_id, "max_score"] = class_score
+
+    user_pred_df = test_metadata.drop_duplicates("observation_id", keep="first")
+    user_pred_df[["observation_id", "class_id"]].to_csv(output_csv_path, index=None)
 
 
 if __name__ == "__main__":
@@ -47,6 +113,14 @@ if __name__ == "__main__":
     with zipfile.ZipFile("/tmp/data/private_testset.zip", 'r') as zip_ref:
         zip_ref.extractall("/tmp/data")
 
+    MODEL_PATH = "swinv2_base_w24_b16x8-fp16_fungi+val2_res_384_genus-loss_no-unknown-target-zero_metadata_epochs_4_epoch_2_20240524-a429ecac.pth"
+    MODEL_NAME = "swinv2_base_w24_b16x8-fp16_fungi+val2_res_384_genus-loss_no-unknown-target-zero_metadata_epochs_4.py"
+
     metadata_file_path = "./FungiCLEF2024_TestMetadata.csv"
+    test_metadata = pd.read_csv(metadata_file_path)
 
-    run_inference(metadata_file_path, "./submission.csv", "/tmp/data/private_testset/")
+    make_submission(
+        test_metadata=test_metadata,
+        model_path=MODEL_PATH,
+        model_name=MODEL_NAME
+    )