#!/usr/bin/env python3 # Copyright (c) Megvii Inc. All rights reserved. import inspect import os import sys from collections import defaultdict from loguru import logger import cv2 import numpy as np import torch def get_caller_name(depth=0): """ Args: depth (int): Depth of caller conext, use 0 for caller depth. Default value: 0. Returns: str: module name of the caller """ # the following logic is a little bit faster than inspect.stack() logic frame = inspect.currentframe().f_back for _ in range(depth): frame = frame.f_back return frame.f_globals["__name__"] class StreamToLoguru: """ stream object that redirects writes to a logger instance. """ def __init__(self, level="INFO", caller_names=("apex", "pycocotools")): """ Args: level(str): log level string of loguru. Default value: "INFO". caller_names(tuple): caller names of redirected module. Default value: (apex, pycocotools). """ self.level = level self.linebuf = "" self.caller_names = caller_names def write(self, buf): full_name = get_caller_name(depth=1) module_name = full_name.rsplit(".", maxsplit=-1)[0] if module_name in self.caller_names: for line in buf.rstrip().splitlines(): # use caller level log logger.opt(depth=2).log(self.level, line.rstrip()) else: sys.__stdout__.write(buf) def flush(self): # flush is related with CPR(cursor position report) in terminal return sys.__stdout__.flush() def isatty(self): # when using colab, jax is installed by default and issue like # https://github.com/Megvii-BaseDetection/YOLOX/issues/1437 might be raised # due to missing attribute like`isatty`. # For more details, checked the following link: # https://github.com/google/jax/blob/10720258ea7fb5bde997dfa2f3f71135ab7a6733/jax/_src/pretty_printer.py#L54 # noqa return sys.__stdout__.isatty() def fileno(self): # To solve the issue when using debug tools like pdb return sys.__stdout__.fileno() def redirect_sys_output(log_level="INFO"): redirect_logger = StreamToLoguru(log_level) sys.stderr = redirect_logger sys.stdout = redirect_logger def setup_logger(save_dir, distributed_rank=0, filename="log.txt", mode="a"): """setup logger for training and testing. Args: save_dir(str): location to save log file distributed_rank(int): device rank when multi-gpu environment filename (string): log save name. mode(str): log file write mode, `append` or `override`. default is `a`. Return: logger instance. """ loguru_format = ( "{time:YYYY-MM-DD HH:mm:ss} | " "{level: <8} | " "{name}:{line} - {message}" ) logger.remove() save_file = os.path.join(save_dir, filename) if mode == "o" and os.path.exists(save_file): os.remove(save_file) # only keep logger in rank0 process if distributed_rank == 0: logger.add( sys.stderr, format=loguru_format, level="INFO", enqueue=True, ) logger.add(save_file) # redirect stdout/stderr to loguru redirect_sys_output("INFO") class WandbLogger(object): """ Log training runs, datasets, models, and predictions to Weights & Biases. This logger sends information to W&B at wandb.ai. By default, this information includes hyperparameters, system configuration and metrics, model metrics, and basic data metrics and analyses. For more information, please refer to: https://docs.wandb.ai/guides/track https://docs.wandb.ai/guides/integrations/other/yolox """ def __init__(self, project=None, name=None, id=None, entity=None, save_dir=None, config=None, val_dataset=None, num_eval_images=100, log_checkpoints=False, **kwargs): """ Args: project (str): wandb project name. name (str): wandb run name. id (str): wandb run id. entity (str): wandb entity name. save_dir (str): save directory. config (dict): config dict. val_dataset (Dataset): validation dataset. num_eval_images (int): number of images from the validation set to log. log_checkpoints (bool): log checkpoints **kwargs: other kwargs. Usage: Any arguments for wandb.init can be provided on the command line using the prefix `wandb-`. Example ``` python tools/train.py .... --logger wandb wandb-project \ wandb-name \ wandb-id \ wandb-save_dir \ wandb-num_eval_imges \ wandb-log_checkpoints ``` The val_dataset argument is not open to the command line. """ try: import wandb self.wandb = wandb except ModuleNotFoundError: raise ModuleNotFoundError( "wandb is not installed." "Please install wandb using pip install wandb" ) from yolox.data.datasets import VOCDetection self.project = project self.name = name self.id = id self.save_dir = save_dir self.config = config self.kwargs = kwargs self.entity = entity self._run = None self.val_artifact = None if num_eval_images == -1: self.num_log_images = len(val_dataset) else: self.num_log_images = min(num_eval_images, len(val_dataset)) self.log_checkpoints = (log_checkpoints == "True" or log_checkpoints == "true") self._wandb_init = dict( project=self.project, name=self.name, id=self.id, entity=self.entity, dir=self.save_dir, resume="allow" ) self._wandb_init.update(**kwargs) _ = self.run if self.config: self.run.config.update(self.config) self.run.define_metric("train/epoch") self.run.define_metric("val/*", step_metric="train/epoch") self.run.define_metric("train/step") self.run.define_metric("train/*", step_metric="train/step") self.voc_dataset = VOCDetection if val_dataset and self.num_log_images != 0: self.val_dataset = val_dataset self.cats = val_dataset.cats self.id_to_class = { cls['id']: cls['name'] for cls in self.cats } self._log_validation_set(val_dataset) @property def run(self): if self._run is None: if self.wandb.run is not None: logger.info( "There is a wandb run already in progress " "and newly created instances of `WandbLogger` will reuse" " this run. If this is not desired, call `wandb.finish()`" "before instantiating `WandbLogger`." ) self._run = self.wandb.run else: self._run = self.wandb.init(**self._wandb_init) return self._run def _log_validation_set(self, val_dataset): """ Log validation set to wandb. Args: val_dataset (Dataset): validation dataset. """ if self.val_artifact is None: self.val_artifact = self.wandb.Artifact(name="validation_images", type="dataset") self.val_table = self.wandb.Table(columns=["id", "input"]) for i in range(self.num_log_images): data_point = val_dataset[i] img = data_point[0] id = data_point[3] img = np.transpose(img, (1, 2, 0)) img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) if isinstance(id, torch.Tensor): id = id.item() self.val_table.add_data( id, self.wandb.Image(img) ) self.val_artifact.add(self.val_table, "validation_images_table") self.run.use_artifact(self.val_artifact) self.val_artifact.wait() def _convert_prediction_format(self, predictions): image_wise_data = defaultdict(int) for key, val in predictions.items(): img_id = key try: bboxes, cls, scores = val except KeyError: bboxes, cls, scores = val["bboxes"], val["categories"], val["scores"] # These store information of actual bounding boxes i.e. the ones which are not None act_box = [] act_scores = [] act_cls = [] if bboxes is not None: for box, classes, score in zip(bboxes, cls, scores): if box is None or score is None or classes is None: continue act_box.append(box) act_scores.append(score) act_cls.append(classes) image_wise_data.update({ int(img_id): { "bboxes": [box.numpy().tolist() for box in act_box], "scores": [score.numpy().item() for score in act_scores], "categories": [ self.val_dataset.class_ids[int(act_cls[ind])] for ind in range(len(act_box)) ], } }) return image_wise_data def log_metrics(self, metrics, step=None): """ Args: metrics (dict): metrics dict. step (int): step number. """ for k, v in metrics.items(): if isinstance(v, torch.Tensor): metrics[k] = v.item() if step is not None: metrics.update({"train/step": step}) self.run.log(metrics) else: self.run.log(metrics) def log_images(self, predictions): if len(predictions) == 0 or self.val_artifact is None or self.num_log_images == 0: return table_ref = self.val_artifact.get("validation_images_table") columns = ["id", "predicted"] for cls in self.cats: columns.append(cls["name"]) if isinstance(self.val_dataset, self.voc_dataset): predictions = self._convert_prediction_format(predictions) result_table = self.wandb.Table(columns=columns) for idx, val in table_ref.iterrows(): avg_scores = defaultdict(int) num_occurrences = defaultdict(int) id = val[0] if isinstance(id, list): id = id[0] if id in predictions: prediction = predictions[id] boxes = [] for i in range(len(prediction["bboxes"])): bbox = prediction["bboxes"][i] x0 = bbox[0] y0 = bbox[1] x1 = bbox[2] y1 = bbox[3] box = { "position": { "minX": min(x0, x1), "minY": min(y0, y1), "maxX": max(x0, x1), "maxY": max(y0, y1) }, "class_id": prediction["categories"][i], "domain": "pixel" } avg_scores[ self.id_to_class[prediction["categories"][i]] ] += prediction["scores"][i] num_occurrences[self.id_to_class[prediction["categories"][i]]] += 1 boxes.append(box) else: boxes = [] average_class_score = [] for cls in self.cats: if cls["name"] not in num_occurrences: score = 0 else: score = avg_scores[cls["name"]] / num_occurrences[cls["name"]] average_class_score.append(score) result_table.add_data( idx, self.wandb.Image(val[1], boxes={ "prediction": { "box_data": boxes, "class_labels": self.id_to_class } } ), *average_class_score ) self.wandb.log({"val_results/result_table": result_table}) def save_checkpoint(self, save_dir, model_name, is_best, metadata=None): """ Args: save_dir (str): save directory. model_name (str): model name. is_best (bool): whether the model is the best model. metadata (dict): metadata to save corresponding to the checkpoint. """ if not self.log_checkpoints: return if "epoch" in metadata: epoch = metadata["epoch"] else: epoch = None filename = os.path.join(save_dir, model_name + "_ckpt.pth") artifact = self.wandb.Artifact( name=f"run_{self.run.id}_model", type="model", metadata=metadata ) artifact.add_file(filename, name="model_ckpt.pth") aliases = ["latest"] if is_best: aliases.append("best") if epoch: aliases.append(f"epoch-{epoch}") self.run.log_artifact(artifact, aliases=aliases) def finish(self): self.run.finish() @classmethod def initialize_wandb_logger(cls, args, exp, val_dataset): wandb_params = dict() prefix = "wandb-" for k, v in zip(args.opts[0::2], args.opts[1::2]): if k.startswith("wandb-"): try: wandb_params.update({k[len(prefix):]: int(v)}) except ValueError: wandb_params.update({k[len(prefix):]: v}) return cls(config=vars(exp), val_dataset=val_dataset, **wandb_params)