Upload 127 files

fastai/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "2.7.13"
+

fastai/_nbdev.py

@@ -0,0 +1,899 @@
+# AUTOGENERATED BY NBDEV! DO NOT EDIT!
+
+__all__ = ["index", "modules", "custom_doc_links", "git_url"]
+
+index = {"defaults.benchmark": "00_torch_core.ipynb",
+         "setup_cuda": "00_torch_core.ipynb",
+         "subplots": "00_torch_core.ipynb",
+         "show_image": "00_torch_core.ipynb",
+         "show_titled_image": "00_torch_core.ipynb",
+         "show_images": "00_torch_core.ipynb",
+         "ArrayBase": "00_torch_core.ipynb",
+         "ArrayImageBase": "00_torch_core.ipynb",
+         "ArrayImage": "00_torch_core.ipynb",
+         "ArrayImageBW": "00_torch_core.ipynb",
+         "ArrayMask": "00_torch_core.ipynb",
+         "Tensor.__array_eq__": "00_torch_core.ipynb",
+         "tensor": "00_torch_core.ipynb",
+         "set_seed": "00_torch_core.ipynb",
+         "get_random_states": "00_torch_core.ipynb",
+         "set_random_states": "00_torch_core.ipynb",
+         "no_random": "00_torch_core.ipynb",
+         "unsqueeze": "00_torch_core.ipynb",
+         "unsqueeze_": "00_torch_core.ipynb",
+         "apply": "00_torch_core.ipynb",
+         "maybe_gather": "00_torch_core.ipynb",
+         "to_detach": "00_torch_core.ipynb",
+         "to_half": "00_torch_core.ipynb",
+         "to_float": "00_torch_core.ipynb",
+         "defaults.use_cuda": "00_torch_core.ipynb",
+         "default_device": "00_torch_core.ipynb",
+         "to_device": "00_torch_core.ipynb",
+         "to_cpu": "00_torch_core.ipynb",
+         "to_np": "00_torch_core.ipynb",
+         "to_concat": "00_torch_core.ipynb",
+         "Tensor.set_meta": "00_torch_core.ipynb",
+         "Tensor.as_subclass": "00_torch_core.ipynb",
+         "TensorBase": "00_torch_core.ipynb",
+         "TensorImageBase": "00_torch_core.ipynb",
+         "TensorImage": "00_torch_core.ipynb",
+         "TensorImageBW": "00_torch_core.ipynb",
+         "TensorMask": "00_torch_core.ipynb",
+         "TensorFlowField": "00_torch_core.ipynb",
+         "TensorCategory": "00_torch_core.ipynb",
+         "TensorMultiCategory": "00_torch_core.ipynb",
+         "TitledTensorScalar": "00_torch_core.ipynb",
+         "L.tensored": "00_torch_core.ipynb",
+         "L.stack": "00_torch_core.ipynb",
+         "L.cat": "00_torch_core.ipynb",
+         "concat": "00_torch_core.ipynb",
+         "Chunks": "00_torch_core.ipynb",
+         "show_title": "00_torch_core.ipynb",
+         "ShowTitle": "00_torch_core.ipynb",
+         "TitledInt": "00_torch_core.ipynb",
+         "TitledFloat": "00_torch_core.ipynb",
+         "TitledStr": "00_torch_core.ipynb",
+         "TitledTuple": "00_torch_core.ipynb",
+         "TitledStr.truncate": "00_torch_core.ipynb",
+         "pd.DataFrame.__init__": "00_torch_core.ipynb",
+         "get_empty_df": "00_torch_core.ipynb",
+         "display_df": "00_torch_core.ipynb",
+         "get_first": "00_torch_core.ipynb",
+         "one_param": "00_torch_core.ipynb",
+         "item_find": "00_torch_core.ipynb",
+         "find_device": "00_torch_core.ipynb",
+         "find_bs": "00_torch_core.ipynb",
+         "np_func": "00_torch_core.ipynb",
+         "Module": "00_torch_core.ipynb",
+         "get_model": "00_torch_core.ipynb",
+         "one_hot": "00_torch_core.ipynb",
+         "one_hot_decode": "00_torch_core.ipynb",
+         "params": "00_torch_core.ipynb",
+         "trainable_params": "00_torch_core.ipynb",
+         "norm_types": "00_torch_core.ipynb",
+         "norm_bias_params": "00_torch_core.ipynb",
+         "batch_to_samples": "00_torch_core.ipynb",
+         "Tensor.interp_1d": "00_torch_core.ipynb",
+         "Tensor.pca": "00_torch_core.ipynb",
+         "logit": "00_torch_core.ipynb",
+         "num_distrib": "00_torch_core.ipynb",
+         "rank_distrib": "00_torch_core.ipynb",
+         "distrib_barrier": "00_torch_core.ipynb",
+         "Path.save_array": "00_torch_core.ipynb",
+         "Path.load_array": "00_torch_core.ipynb",
+         "base_doc": "00_torch_core.ipynb",
+         "doc": "00_torch_core.ipynb",
+         "nested_reorder": "00_torch_core.ipynb",
+         "make_cross_image": "00_torch_core.ipynb",
+         "show_image_batch": "00_torch_core.ipynb",
+         "requires_grad": "00_torch_core.ipynb",
+         "init_default": "01_layers.ipynb",
+         "cond_init": "00_torch_core.ipynb",
+         "apply_leaf": "00_torch_core.ipynb",
+         "apply_init": "00_torch_core.ipynb",
+         "script_use_ctx": "00_torch_core.ipynb",
+         "script_save_ctx": "00_torch_core.ipynb",
+         "script_fwd": "00_torch_core.ipynb",
+         "script_bwd": "00_torch_core.ipynb",
+         "grad_module": "00_torch_core.ipynb",
+         "ismin_torch": "00_torch_core.ipynb",
+         "notmax_torch": "00_torch_core.ipynb",
+         "module": "01_layers.ipynb",
+         "Identity": "01_layers.ipynb",
+         "Lambda": "01_layers.ipynb",
+         "PartialLambda": "01_layers.ipynb",
+         "Flatten": "01_layers.ipynb",
+         "ToTensorBase": "01_layers.ipynb",
+         "View": "01_layers.ipynb",
+         "ResizeBatch": "01_layers.ipynb",
+         "Debugger": "01_layers.ipynb",
+         "sigmoid_range": "01_layers.ipynb",
+         "SigmoidRange": "01_layers.ipynb",
+         "AdaptiveConcatPool1d": "01_layers.ipynb",
+         "AdaptiveConcatPool2d": "01_layers.ipynb",
+         "PoolType": "01_layers.ipynb",
+         "adaptive_pool": "01_layers.ipynb",
+         "PoolFlatten": "01_layers.ipynb",
+         "NormType": "01_layers.ipynb",
+         "BatchNorm": "01_layers.ipynb",
+         "InstanceNorm": "01_layers.ipynb",
+         "BatchNorm1dFlat": "01_layers.ipynb",
+         "LinBnDrop": "01_layers.ipynb",
+         "sigmoid": "01_layers.ipynb",
+         "sigmoid_": "01_layers.ipynb",
+         "vleaky_relu": "01_layers.ipynb",
+         "init_linear": "01_layers.ipynb",
+         "defaults.activation": "01_layers.ipynb",
+         "ConvLayer": "01_layers.ipynb",
+         "AdaptiveAvgPool": "01_layers.ipynb",
+         "MaxPool": "01_layers.ipynb",
+         "AvgPool": "01_layers.ipynb",
+         "trunc_normal_": "01_layers.ipynb",
+         "Embedding": "01_layers.ipynb",
+         "SelfAttention": "01_layers.ipynb",
+         "PooledSelfAttention2d": "01_layers.ipynb",
+         "SimpleSelfAttention": "01_layers.ipynb",
+         "icnr_init": "01_layers.ipynb",
+         "PixelShuffle_ICNR": "01_layers.ipynb",
+         "sequential": "01_layers.ipynb",
+         "SequentialEx": "01_layers.ipynb",
+         "MergeLayer": "01_layers.ipynb",
+         "Cat": "01_layers.ipynb",
+         "SimpleCNN": "01_layers.ipynb",
+         "ProdLayer": "01_layers.ipynb",
+         "inplace_relu": "01_layers.ipynb",
+         "SEModule": "01_layers.ipynb",
+         "ResBlock": "01_layers.ipynb",
+         "SEBlock": "01_layers.ipynb",
+         "SEResNeXtBlock": "01_layers.ipynb",
+         "SeparableBlock": "01_layers.ipynb",
+         "TimeDistributed": "01_layers.ipynb",
+         "swish": "01_layers.ipynb",
+         "Swish": "01_layers.ipynb",
+         "MishJitAutoFn": "01_layers.ipynb",
+         "mish": "01_layers.ipynb",
+         "Mish": "01_layers.ipynb",
+         "ParameterModule": "01_layers.ipynb",
+         "children_and_parameters": "01_layers.ipynb",
+         "has_children": "01_layers.ipynb",
+         "flatten_model": "01_layers.ipynb",
+         "NoneReduce": "01_layers.ipynb",
+         "in_channels": "01_layers.ipynb",
+         "BaseLoss": "01a_losses.ipynb",
+         "CrossEntropyLossFlat": "01a_losses.ipynb",
+         "FocalLoss": "01a_losses.ipynb",
+         "FocalLossFlat": "01a_losses.ipynb",
+         "BCEWithLogitsLossFlat": "01a_losses.ipynb",
+         "BCELossFlat": "01a_losses.ipynb",
+         "MSELossFlat": "01a_losses.ipynb",
+         "L1LossFlat": "01a_losses.ipynb",
+         "LabelSmoothingCrossEntropy": "01a_losses.ipynb",
+         "LabelSmoothingCrossEntropyFlat": "01a_losses.ipynb",
+         "DiceLoss": "01a_losses.ipynb",
+         "fa_collate": "02_data.load.ipynb",
+         "fa_convert": "02_data.load.ipynb",
+         "SkipItemException": "02_data.load.ipynb",
+         "collate_error": "02_data.load.ipynb",
+         "DataLoader": "02_data.load.ipynb",
+         "TfmdDL": "03_data.core.ipynb",
+         "DataLoaders": "03_data.core.ipynb",
+         "FilteredBase": "03_data.core.ipynb",
+         "TfmdLists": "03_data.core.ipynb",
+         "decode_at": "03_data.core.ipynb",
+         "show_at": "03_data.core.ipynb",
+         "Datasets": "03_data.core.ipynb",
+         "test_set": "03_data.core.ipynb",
+         "DataLoaders.test_dl": "03_data.core.ipynb",
+         "fastai_cfg": "04_data.external.ipynb",
+         "fastai_path": "04_data.external.ipynb",
+         "URLs": "04_data.external.ipynb",
+         "untar_data": "04_data.external.ipynb",
+         "get_files": "05_data.transforms.ipynb",
+         "FileGetter": "05_data.transforms.ipynb",
+         "image_extensions": "05_data.transforms.ipynb",
+         "get_image_files": "05_data.transforms.ipynb",
+         "ImageGetter": "05_data.transforms.ipynb",
+         "get_text_files": "05_data.transforms.ipynb",
+         "ItemGetter": "05_data.transforms.ipynb",
+         "AttrGetter": "05_data.transforms.ipynb",
+         "RandomSplitter": "05_data.transforms.ipynb",
+         "TrainTestSplitter": "05_data.transforms.ipynb",
+         "IndexSplitter": "05_data.transforms.ipynb",
+         "EndSplitter": "05_data.transforms.ipynb",
+         "GrandparentSplitter": "05_data.transforms.ipynb",
+         "FuncSplitter": "05_data.transforms.ipynb",
+         "MaskSplitter": "05_data.transforms.ipynb",
+         "FileSplitter": "05_data.transforms.ipynb",
+         "ColSplitter": "05_data.transforms.ipynb",
+         "RandomSubsetSplitter": "05_data.transforms.ipynb",
+         "parent_label": "05_data.transforms.ipynb",
+         "RegexLabeller": "05_data.transforms.ipynb",
+         "ColReader": "05_data.transforms.ipynb",
+         "CategoryMap": "05_data.transforms.ipynb",
+         "Categorize": "05_data.transforms.ipynb",
+         "Category": "05_data.transforms.ipynb",
+         "MultiCategorize": "05_data.transforms.ipynb",
+         "MultiCategory": "05_data.transforms.ipynb",
+         "OneHotEncode": "05_data.transforms.ipynb",
+         "EncodedMultiCategorize": "05_data.transforms.ipynb",
+         "RegressionSetup": "05_data.transforms.ipynb",
+         "get_c": "05_data.transforms.ipynb",
+         "ToTensor": "05_data.transforms.ipynb",
+         "IntToFloatTensor": "05_data.transforms.ipynb",
+         "broadcast_vec": "05_data.transforms.ipynb",
+         "Normalize": "05_data.transforms.ipynb",
+         "TransformBlock": "06_data.block.ipynb",
+         "CategoryBlock": "06_data.block.ipynb",
+         "MultiCategoryBlock": "06_data.block.ipynb",
+         "RegressionBlock": "06_data.block.ipynb",
+         "DataBlock": "06_data.block.ipynb",
+         "DataBlock.summary": "06_data.block.ipynb",
+         "imagenet_stats": "07_vision.core.ipynb",
+         "cifar_stats": "07_vision.core.ipynb",
+         "mnist_stats": "07_vision.core.ipynb",
+         "n_px": "07_vision.core.ipynb",
+         "shape": "60_medical.imaging.ipynb",
+         "aspect": "07_vision.core.ipynb",
+         "Image.Image.reshape": "07_vision.core.ipynb",
+         "Image.Image.to_bytes_format": "07_vision.core.ipynb",
+         "Image.Image.to_thumb": "07_vision.core.ipynb",
+         "Image.Image.resize_max": "07_vision.core.ipynb",
+         "to_image": "07_vision.core.ipynb",
+         "load_image": "07_vision.core.ipynb",
+         "image2tensor": "07_vision.core.ipynb",
+         "PILBase": "07_vision.core.ipynb",
+         "PILImage": "07_vision.core.ipynb",
+         "PILImageBW": "07_vision.core.ipynb",
+         "PILMask": "07_vision.core.ipynb",
+         "OpenMask": "07_vision.core.ipynb",
+         "OpenMask.loss_func": "07_vision.core.ipynb",
+         "PILMask.create": "07_vision.core.ipynb",
+         "AddMaskCodes": "07_vision.core.ipynb",
+         "TensorPoint": "07_vision.core.ipynb",
+         "TensorPointCreate": "07_vision.core.ipynb",
+         "TensorPointCreate.loss_func": "07_vision.core.ipynb",
+         "TensorPoint.create": "07_vision.core.ipynb",
+         "get_annotations": "07_vision.core.ipynb",
+         "TensorBBox": "07_vision.core.ipynb",
+         "LabeledBBox": "07_vision.core.ipynb",
+         "encodes": "40_tabular.core.ipynb",
+         "PointScaler": "07_vision.core.ipynb",
+         "BBoxLabeler": "07_vision.core.ipynb",
+         "decodes": "40_tabular.core.ipynb",
+         "get_grid": "08_vision.data.ipynb",
+         "clip_remove_empty": "08_vision.data.ipynb",
+         "bb_pad": "08_vision.data.ipynb",
+         "ImageBlock": "08_vision.data.ipynb",
+         "MaskBlock": "08_vision.data.ipynb",
+         "PointBlock": "08_vision.data.ipynb",
+         "BBoxBlock": "08_vision.data.ipynb",
+         "PointBlock.__doc__": "08_vision.data.ipynb",
+         "BBoxBlock.__doc__": "08_vision.data.ipynb",
+         "BBoxLblBlock": "08_vision.data.ipynb",
+         "ImageDataLoaders": "08_vision.data.ipynb",
+         "ImageDataLoaders.from_csv": "08_vision.data.ipynb",
+         "ImageDataLoaders.from_name_func": "08_vision.data.ipynb",
+         "ImageDataLoaders.from_path_re": "08_vision.data.ipynb",
+         "ImageDataLoaders.from_name_re": "08_vision.data.ipynb",
+         "SegmentationDataLoaders": "08_vision.data.ipynb",
+         "RandTransform": "09_vision.augment.ipynb",
+         "TensorTypes": "09_vision.augment.ipynb",
+         "Image.Image.flip_lr": "09_vision.augment.ipynb",
+         "TensorImageBase.flip_lr": "09_vision.augment.ipynb",
+         "TensorPoint.flip_lr": "09_vision.augment.ipynb",
+         "TensorBBox.flip_lr": "09_vision.augment.ipynb",
+         "FlipItem": "09_vision.augment.ipynb",
+         "PILImage.dihedral": "09_vision.augment.ipynb",
+         "TensorImage.dihedral": "09_vision.augment.ipynb",
+         "TensorPoint.dihedral": "09_vision.augment.ipynb",
+         "TensorBBox.dihedral": "09_vision.augment.ipynb",
+         "DihedralItem": "09_vision.augment.ipynb",
+         "TensorBBox.crop_pad": "09_vision.augment.ipynb",
+         "TensorPoint.crop_pad": "09_vision.augment.ipynb",
+         "Image.Image.crop_pad": "09_vision.augment.ipynb",
+         "CropPad": "09_vision.augment.ipynb",
+         "RandomCrop": "09_vision.augment.ipynb",
+         "OldRandomCrop": "09_vision.augment.ipynb",
+         "Resize": "09_vision.augment.ipynb",
+         "RandomResizedCrop": "09_vision.augment.ipynb",
+         "RatioResize": "09_vision.augment.ipynb",
+         "affine_grid": "09_vision.augment.ipynb",
+         "TensorImage.affine_coord": "09_vision.augment.ipynb",
+         "TensorMask.affine_coord": "09_vision.augment.ipynb",
+         "TensorPoint.affine_coord": "09_vision.augment.ipynb",
+         "TensorBBox.affine_coord": "09_vision.augment.ipynb",
+         "AffineCoordTfm": "09_vision.augment.ipynb",
+         "RandomResizedCropGPU": "09_vision.augment.ipynb",
+         "mask_tensor": "09_vision.augment.ipynb",
+         "affine_mat": "09_vision.augment.ipynb",
+         "flip_mat": "09_vision.augment.ipynb",
+         "TensorImage.flip_batch": "09_vision.augment.ipynb",
+         "TensorMask.flip_batch": "09_vision.augment.ipynb",
+         "TensorPoint.flip_batch": "09_vision.augment.ipynb",
+         "TensorBBox.flip_batch": "09_vision.augment.ipynb",
+         "Flip": "09_vision.augment.ipynb",
+         "DeterministicDraw": "09_vision.augment.ipynb",
+         "DeterministicFlip": "09_vision.augment.ipynb",
+         "dihedral_mat": "09_vision.augment.ipynb",
+         "TensorImage.dihedral_batch": "09_vision.augment.ipynb",
+         "TensorMask.dihedral_batch": "09_vision.augment.ipynb",
+         "TensorPoint.dihedral_batch": "09_vision.augment.ipynb",
+         "TensorBBox.dihedral_batch": "09_vision.augment.ipynb",
+         "Dihedral": "09_vision.augment.ipynb",
+         "DeterministicDihedral": "09_vision.augment.ipynb",
+         "rotate_mat": "09_vision.augment.ipynb",
+         "TensorImage.rotate": "09_vision.augment.ipynb",
+         "TensorMask.rotate": "09_vision.augment.ipynb",
+         "TensorPoint.rotate": "09_vision.augment.ipynb",
+         "TensorBBox.rotate": "09_vision.augment.ipynb",
+         "Rotate": "09_vision.augment.ipynb",
+         "zoom_mat": "09_vision.augment.ipynb",
+         "TensorImage.zoom": "09_vision.augment.ipynb",
+         "TensorMask.zoom": "09_vision.augment.ipynb",
+         "TensorPoint.zoom": "09_vision.augment.ipynb",
+         "TensorBBox.zoom": "09_vision.augment.ipynb",
+         "Zoom": "09_vision.augment.ipynb",
+         "find_coeffs": "09_vision.augment.ipynb",
+         "apply_perspective": "09_vision.augment.ipynb",
+         "TensorImage.warp": "09_vision.augment.ipynb",
+         "TensorMask.warp": "09_vision.augment.ipynb",
+         "TensorPoint.warp": "09_vision.augment.ipynb",
+         "TensorBBox.warp": "09_vision.augment.ipynb",
+         "Warp": "09_vision.augment.ipynb",
+         "TensorImage.lighting": "09_vision.augment.ipynb",
+         "SpaceTfm": "09_vision.augment.ipynb",
+         "LightingTfm": "09_vision.augment.ipynb",
+         "TensorImage.brightness": "09_vision.augment.ipynb",
+         "Brightness": "09_vision.augment.ipynb",
+         "TensorImage.contrast": "09_vision.augment.ipynb",
+         "Contrast": "09_vision.augment.ipynb",
+         "grayscale": "09_vision.augment.ipynb",
+         "TensorImage.saturation": "09_vision.augment.ipynb",
+         "Saturation": "09_vision.augment.ipynb",
+         "rgb2hsv": "09_vision.augment.ipynb",
+         "hsv2rgb": "09_vision.augment.ipynb",
+         "TensorImage.hsv": "09_vision.augment.ipynb",
+         "HSVTfm": "09_vision.augment.ipynb",
+         "TensorImage.hue": "09_vision.augment.ipynb",
+         "Hue": "09_vision.augment.ipynb",
+         "cutout_gaussian": "09_vision.augment.ipynb",
+         "norm_apply_denorm": "09_vision.augment.ipynb",
+         "RandomErasing": "09_vision.augment.ipynb",
+         "setup_aug_tfms": "09_vision.augment.ipynb",
+         "aug_transforms": "09_vision.augment.ipynb",
+         "download_images": "09b_vision.utils.ipynb",
+         "resize_to": "09b_vision.utils.ipynb",
+         "verify_image": "09b_vision.utils.ipynb",
+         "verify_images": "09b_vision.utils.ipynb",
+         "resize_image": "09b_vision.utils.ipynb",
+         "resize_images": "09b_vision.utils.ipynb",
+         "Box.__getitem__": "09c_vision.widgets.ipynb",
+         "widget": "09c_vision.widgets.ipynb",
+         "carousel": "09c_vision.widgets.ipynb",
+         "ImagesCleaner": "09c_vision.widgets.ipynb",
+         "ImageClassifierCleaner": "09c_vision.widgets.ipynb",
+         "init_cnn": "11_vision.models.xresnet.ipynb",
+         "XResNet": "11_vision.models.xresnet.ipynb",
+         "xresnet18": "11_vision.models.xresnet.ipynb",
+         "xresnet34": "11_vision.models.xresnet.ipynb",
+         "xresnet50": "11_vision.models.xresnet.ipynb",
+         "xresnet101": "11_vision.models.xresnet.ipynb",
+         "xresnet152": "11_vision.models.xresnet.ipynb",
+         "xresnet18_deep": "11_vision.models.xresnet.ipynb",
+         "xresnet34_deep": "11_vision.models.xresnet.ipynb",
+         "xresnet50_deep": "11_vision.models.xresnet.ipynb",
+         "xresnet18_deeper": "11_vision.models.xresnet.ipynb",
+         "xresnet34_deeper": "11_vision.models.xresnet.ipynb",
+         "xresnet50_deeper": "11_vision.models.xresnet.ipynb",
+         "se_kwargs1": "11_vision.models.xresnet.ipynb",
+         "se_kwargs2": "11_vision.models.xresnet.ipynb",
+         "se_kwargs3": "11_vision.models.xresnet.ipynb",
+         "g0": "11_vision.models.xresnet.ipynb",
+         "g1": "11_vision.models.xresnet.ipynb",
+         "g2": "11_vision.models.xresnet.ipynb",
+         "g3": "11_vision.models.xresnet.ipynb",
+         "xse_resnet18": "11_vision.models.xresnet.ipynb",
+         "xse_resnext18": "11_vision.models.xresnet.ipynb",
+         "xresnext18": "11_vision.models.xresnet.ipynb",
+         "xse_resnet34": "11_vision.models.xresnet.ipynb",
+         "xse_resnext34": "11_vision.models.xresnet.ipynb",
+         "xresnext34": "11_vision.models.xresnet.ipynb",
+         "xse_resnet50": "11_vision.models.xresnet.ipynb",
+         "xse_resnext50": "11_vision.models.xresnet.ipynb",
+         "xresnext50": "11_vision.models.xresnet.ipynb",
+         "xse_resnet101": "11_vision.models.xresnet.ipynb",
+         "xse_resnext101": "11_vision.models.xresnet.ipynb",
+         "xresnext101": "11_vision.models.xresnet.ipynb",
+         "xse_resnet152": "11_vision.models.xresnet.ipynb",
+         "xsenet154": "11_vision.models.xresnet.ipynb",
+         "xse_resnext18_deep": "11_vision.models.xresnet.ipynb",
+         "xse_resnext34_deep": "11_vision.models.xresnet.ipynb",
+         "xse_resnext50_deep": "11_vision.models.xresnet.ipynb",
+         "xse_resnext18_deeper": "11_vision.models.xresnet.ipynb",
+         "xse_resnext34_deeper": "11_vision.models.xresnet.ipynb",
+         "xse_resnext50_deeper": "11_vision.models.xresnet.ipynb",
+         "Optimizer": "12_optimizer.ipynb",
+         "sgd_step": "12_optimizer.ipynb",
+         "weight_decay": "12_optimizer.ipynb",
+         "weight_decay.defaults": "12_optimizer.ipynb",
+         "l2_reg": "12_optimizer.ipynb",
+         "l2_reg.defaults": "12_optimizer.ipynb",
+         "average_grad": "12_optimizer.ipynb",
+         "average_grad.defaults": "12_optimizer.ipynb",
+         "average_sqr_grad": "12_optimizer.ipynb",
+         "average_sqr_grad.defaults": "12_optimizer.ipynb",
+         "momentum_step": "12_optimizer.ipynb",
+         "SGD": "12_optimizer.ipynb",
+         "rms_prop_step": "12_optimizer.ipynb",
+         "rms_prop_step.defaults": "12_optimizer.ipynb",
+         "RMSProp": "12_optimizer.ipynb",
+         "step_stat": "12_optimizer.ipynb",
+         "debias": "12_optimizer.ipynb",
+         "adam_step": "12_optimizer.ipynb",
+         "Adam": "12_optimizer.ipynb",
+         "radam_step": "12_optimizer.ipynb",
+         "RAdam": "12_optimizer.ipynb",
+         "qhadam_step": "12_optimizer.ipynb",
+         "QHAdam": "12_optimizer.ipynb",
+         "larc_layer_lr": "12_optimizer.ipynb",
+         "larc_layer_lr.defaults": "12_optimizer.ipynb",
+         "larc_step": "12_optimizer.ipynb",
+         "Larc": "12_optimizer.ipynb",
+         "lamb_step": "12_optimizer.ipynb",
+         "Lamb": "12_optimizer.ipynb",
+         "Lookahead": "12_optimizer.ipynb",
+         "ranger": "12_optimizer.ipynb",
+         "detuplify_pg": "12_optimizer.ipynb",
+         "set_item_pg": "12_optimizer.ipynb",
+         "pytorch_hp_map": "12_optimizer.ipynb",
+         "OptimWrapper": "12_optimizer.ipynb",
+         "Callback": "13_callback.core.ipynb",
+         "TrainEvalCallback": "13_callback.core.ipynb",
+         "GatherPredsCallback": "13_callback.core.ipynb",
+         "FetchPredsCallback": "13_callback.core.ipynb",
+         "defaults.lr": "13a_learner.ipynb",
+         "replacing_yield": "13a_learner.ipynb",
+         "mk_metric": "13a_learner.ipynb",
+         "save_model": "13a_learner.ipynb",
+         "load_model": "13a_learner.ipynb",
+         "SkipToEpoch": "13a_learner.ipynb",
+         "Learner": "13a_learner.ipynb",
+         "before_batch_cb": "13a_learner.ipynb",
+         "Learner.save": "13a_learner.ipynb",
+         "Learner.load": "13a_learner.ipynb",
+         "Learner.export": "13a_learner.ipynb",
+         "load_learner": "13a_learner.ipynb",
+         "Metric": "13a_learner.ipynb",
+         "AvgMetric": "13a_learner.ipynb",
+         "AvgLoss": "13a_learner.ipynb",
+         "AvgSmoothLoss": "13a_learner.ipynb",
+         "ValueMetric": "13a_learner.ipynb",
+         "Recorder": "13a_learner.ipynb",
+         "CastToTensor": "13a_learner.ipynb",
+         "Learner.freeze_to": "13a_learner.ipynb",
+         "Learner.freeze": "13a_learner.ipynb",
+         "Learner.unfreeze": "13a_learner.ipynb",
+         "Learner.tta": "13a_learner.ipynb",
+         "flatten_check": "13b_metrics.ipynb",
+         "AccumMetric": "13b_metrics.ipynb",
+         "skm_to_fastai": "13b_metrics.ipynb",
+         "optim_metric": "13b_metrics.ipynb",
+         "accuracy": "13b_metrics.ipynb",
+         "error_rate": "13b_metrics.ipynb",
+         "top_k_accuracy": "13b_metrics.ipynb",
+         "APScoreBinary": "13b_metrics.ipynb",
+         "BalancedAccuracy": "13b_metrics.ipynb",
+         "BrierScore": "13b_metrics.ipynb",
+         "CohenKappa": "13b_metrics.ipynb",
+         "F1Score": "13b_metrics.ipynb",
+         "FBeta": "13b_metrics.ipynb",
+         "HammingLoss": "13b_metrics.ipynb",
+         "Jaccard": "13b_metrics.ipynb",
+         "Precision": "13b_metrics.ipynb",
+         "Recall": "13b_metrics.ipynb",
+         "RocAuc": "13b_metrics.ipynb",
+         "RocAucBinary": "13b_metrics.ipynb",
+         "MatthewsCorrCoef": "13b_metrics.ipynb",
+         "accuracy_multi": "13b_metrics.ipynb",
+         "APScoreMulti": "13b_metrics.ipynb",
+         "BrierScoreMulti": "13b_metrics.ipynb",
+         "F1ScoreMulti": "13b_metrics.ipynb",
+         "FBetaMulti": "13b_metrics.ipynb",
+         "HammingLossMulti": "13b_metrics.ipynb",
+         "JaccardMulti": "13b_metrics.ipynb",
+         "MatthewsCorrCoefMulti": "13b_metrics.ipynb",
+         "PrecisionMulti": "13b_metrics.ipynb",
+         "RecallMulti": "13b_metrics.ipynb",
+         "RocAucMulti": "13b_metrics.ipynb",
+         "mse": "13b_metrics.ipynb",
+         "rmse": "13b_metrics.ipynb",
+         "rmse.__doc__": "13b_metrics.ipynb",
+         "mae": "13b_metrics.ipynb",
+         "msle": "13b_metrics.ipynb",
+         "exp_rmspe": "13b_metrics.ipynb",
+         "exp_rmspe.__doc__": "13b_metrics.ipynb",
+         "ExplainedVariance": "13b_metrics.ipynb",
+         "R2Score": "13b_metrics.ipynb",
+         "PearsonCorrCoef": "13b_metrics.ipynb",
+         "SpearmanCorrCoef": "13b_metrics.ipynb",
+         "foreground_acc": "13b_metrics.ipynb",
+         "Dice": "13b_metrics.ipynb",
+         "DiceMulti": "13b_metrics.ipynb",
+         "JaccardCoeff": "13b_metrics.ipynb",
+         "CorpusBLEUMetric": "13b_metrics.ipynb",
+         "Perplexity": "13b_metrics.ipynb",
+         "perplexity": "13b_metrics.ipynb",
+         "LossMetric": "13b_metrics.ipynb",
+         "LossMetrics": "13b_metrics.ipynb",
+         "annealer": "14_callback.schedule.ipynb",
+         "sched_lin": "14_callback.schedule.ipynb",
+         "sched_cos": "14_callback.schedule.ipynb",
+         "sched_no": "14_callback.schedule.ipynb",
+         "sched_exp": "14_callback.schedule.ipynb",
+         "SchedLin": "14_callback.schedule.ipynb",
+         "SchedCos": "14_callback.schedule.ipynb",
+         "SchedNo": "14_callback.schedule.ipynb",
+         "SchedExp": "14_callback.schedule.ipynb",
+         "SchedLin.__doc__": "14_callback.schedule.ipynb",
+         "SchedCos.__doc__": "14_callback.schedule.ipynb",
+         "SchedExp.__doc__": "14_callback.schedule.ipynb",
+         "SchedPoly": "14_callback.schedule.ipynb",
+         "combine_scheds": "14_callback.schedule.ipynb",
+         "combined_cos": "14_callback.schedule.ipynb",
+         "ParamScheduler": "14_callback.schedule.ipynb",
+         "Learner.fit_one_cycle": "14_callback.schedule.ipynb",
+         "Recorder.plot_sched": "14_callback.schedule.ipynb",
+         "Learner.fit_flat_cos": "14_callback.schedule.ipynb",
+         "Learner.fit_sgdr": "14_callback.schedule.ipynb",
+         "Learner.fine_tune": "14_callback.schedule.ipynb",
+         "LRFinder": "14_callback.schedule.ipynb",
+         "valley": "14_callback.schedule.ipynb",
+         "slide": "14_callback.schedule.ipynb",
+         "minimum": "14_callback.schedule.ipynb",
+         "steep": "14_callback.schedule.ipynb",
+         "Recorder.plot_lr_find": "14_callback.schedule.ipynb",
+         "Learner.lr_find": "14_callback.schedule.ipynb",
+         "CollectDataCallback": "14a_callback.data.ipynb",
+         "WeightedDL": "14a_callback.data.ipynb",
+         "Datasets.weighted_dataloaders": "14a_callback.data.ipynb",
+         "DataBlock.weighted_dataloaders": "14a_callback.data.ipynb",
+         "PartialDL": "14a_callback.data.ipynb",
+         "FilteredBase.partial_dataloaders": "14a_callback.data.ipynb",
+         "Hook": "15_callback.hook.ipynb",
+         "hook_output": "15_callback.hook.ipynb",
+         "Hooks": "15_callback.hook.ipynb",
+         "hook_outputs": "15_callback.hook.ipynb",
+         "dummy_eval": "15_callback.hook.ipynb",
+         "model_sizes": "15_callback.hook.ipynb",
+         "num_features_model": "15_callback.hook.ipynb",
+         "has_params": "15_callback.hook.ipynb",
+         "HookCallback": "15_callback.hook.ipynb",
+         "total_params": "15_callback.hook.ipynb",
+         "layer_info": "15_callback.hook.ipynb",
+         "module_summary": "15_callback.hook.ipynb",
+         "Learner.summary": "15_callback.hook.ipynb",
+         "ActivationStats": "15_callback.hook.ipynb",
+         "UnetBlock": "15a_vision.models.unet.ipynb",
+         "ResizeToOrig": "15a_vision.models.unet.ipynb",
+         "DynamicUnet": "15a_vision.models.unet.ipynb",
+         "ProgressCallback": "16_callback.progress.ipynb",
+         "Learner.no_bar": "16_callback.progress.ipynb",
+         "ShowGraphCallback": "16_callback.progress.ipynb",
+         "CSVLogger": "16_callback.progress.ipynb",
+         "TerminateOnNaNCallback": "17_callback.tracker.ipynb",
+         "TrackerCallback": "17_callback.tracker.ipynb",
+         "EarlyStoppingCallback": "17_callback.tracker.ipynb",
+         "SaveModelCallback": "17_callback.tracker.ipynb",
+         "ReduceLROnPlateau": "17_callback.tracker.ipynb",
+         "MixedPrecision": "18_callback.fp16.ipynb",
+         "FP16TestCallback": "18_callback.fp16.ipynb",
+         "Learner.to_fp16": "18_callback.fp16.ipynb",
+         "Learner.to_fp32": "18_callback.fp16.ipynb",
+         "get_master": "18_callback.fp16.ipynb",
+         "to_master_grads": "18_callback.fp16.ipynb",
+         "to_model_params": "18_callback.fp16.ipynb",
+         "test_overflow": "18_callback.fp16.ipynb",
+         "grad_overflow": "18_callback.fp16.ipynb",
+         "copy_clone": "18_callback.fp16.ipynb",
+         "ModelToHalf": "18_callback.fp16.ipynb",
+         "NonNativeMixedPrecision": "18_callback.fp16.ipynb",
+         "Learner.to_non_native_fp16": "18_callback.fp16.ipynb",
+         "Learner.to_non_native_fp32": "18_callback.fp16.ipynb",
+         "ShortEpochCallback": "18a_callback.training.ipynb",
+         "GradientAccumulation": "18a_callback.training.ipynb",
+         "GradientClip": "18a_callback.training.ipynb",
+         "set_bn_eval": "18a_callback.training.ipynb",
+         "BnFreeze": "18a_callback.training.ipynb",
+         "bn_types": "18a_callback.training.ipynb",
+         "ChannelsLast": "18a_callback.training.ipynb",
+         "MCDropoutCallback": "18b_callback.preds.ipynb",
+         "reduce_loss": "19_callback.mixup.ipynb",
+         "MixHandler": "19_callback.mixup.ipynb",
+         "MixUp": "19_callback.mixup.ipynb",
+         "CutMix": "19_callback.mixup.ipynb",
+         "Interpretation": "20_interpret.ipynb",
+         "ClassificationInterpretation": "20_interpret.ipynb",
+         "SegmentationInterpretation": "20_interpret.ipynb",
+         "DataParallel.reset": "20a_distributed.ipynb",
+         "ParallelTrainer": "20a_distributed.ipynb",
+         "Learner.to_parallel": "20a_distributed.ipynb",
+         "Learner.detach_parallel": "20a_distributed.ipynb",
+         "Learner.parallel_ctx": "20a_distributed.ipynb",
+         "DistributedDataParallel.reset": "20a_distributed.ipynb",
+         "setup_distrib": "20a_distributed.ipynb",
+         "teardown_distrib": "20a_distributed.ipynb",
+         "DistributedDL": "20a_distributed.ipynb",
+         "DistributedTrainer": "20a_distributed.ipynb",
+         "Learner.to_distributed": "20a_distributed.ipynb",
+         "Learner.detach_distributed": "20a_distributed.ipynb",
+         "Learner.distrib_ctx": "20a_distributed.ipynb",
+         "rank0_first": "20a_distributed.ipynb",
+         "has_pool_type": "21_vision.learner.ipynb",
+         "cut_model": "21_vision.learner.ipynb",
+         "create_body": "21_vision.learner.ipynb",
+         "create_head": "21_vision.learner.ipynb",
+         "default_split": "21_vision.learner.ipynb",
+         "model_meta": "21_vision.learner.ipynb",
+         "add_head": "21_vision.learner.ipynb",
+         "create_vision_model": "21_vision.learner.ipynb",
+         "TimmBody": "21_vision.learner.ipynb",
+         "create_timm_model": "21_vision.learner.ipynb",
+         "vision_learner": "21_vision.learner.ipynb",
+         "create_unet_model": "21_vision.learner.ipynb",
+         "unet_learner": "21_vision.learner.ipynb",
+         "create_cnn_model": "21_vision.learner.ipynb",
+         "cnn_learner": "21_vision.learner.ipynb",
+         "GANModule": "24_vision.gan.ipynb",
+         "basic_critic": "24_vision.gan.ipynb",
+         "AddChannels": "24_vision.gan.ipynb",
+         "basic_generator": "24_vision.gan.ipynb",
+         "DenseResBlock": "24_vision.gan.ipynb",
+         "gan_critic": "24_vision.gan.ipynb",
+         "GANLoss": "24_vision.gan.ipynb",
+         "AdaptiveLoss": "24_vision.gan.ipynb",
+         "accuracy_thresh_expand": "24_vision.gan.ipynb",
+         "set_freeze_model": "24_vision.gan.ipynb",
+         "GANTrainer": "24_vision.gan.ipynb",
+         "FixedGANSwitcher": "24_vision.gan.ipynb",
+         "AdaptiveGANSwitcher": "24_vision.gan.ipynb",
+         "GANDiscriminativeLR": "24_vision.gan.ipynb",
+         "InvisibleTensor": "24_vision.gan.ipynb",
+         "generate_noise": "24_vision.gan.ipynb",
+         "gan_loss_from_func": "24_vision.gan.ipynb",
+         "GANLearner": "24_vision.gan.ipynb",
+         "GANLearner.from_learners": "24_vision.gan.ipynb",
+         "GANLearner.wgan": "24_vision.gan.ipynb",
+         "spec_add_spaces": "30_text.core.ipynb",
+         "rm_useless_spaces": "30_text.core.ipynb",
+         "replace_rep": "30_text.core.ipynb",
+         "replace_wrep": "30_text.core.ipynb",
+         "fix_html": "30_text.core.ipynb",
+         "replace_all_caps": "30_text.core.ipynb",
+         "replace_maj": "30_text.core.ipynb",
+         "lowercase": "30_text.core.ipynb",
+         "replace_space": "30_text.core.ipynb",
+         "defaults.text_spec_tok": "30_text.core.ipynb",
+         "defaults.text_proc_rules": "30_text.core.ipynb",
+         "defaults.text_postproc_rules": "30_text.core.ipynb",
+         "BaseTokenizer": "30_text.core.ipynb",
+         "SpacyTokenizer": "30_text.core.ipynb",
+         "WordTokenizer": "30_text.core.ipynb",
+         "TokenizeWithRules": "30_text.core.ipynb",
+         "tokenize1": "30_text.core.ipynb",
+         "parallel_tokenize": "30_text.core.ipynb",
+         "fn_counter_pkl": "30_text.core.ipynb",
+         "fn_lengths_pkl": "30_text.core.ipynb",
+         "tokenize_folder": "30_text.core.ipynb",
+         "tokenize_files": "30_text.core.ipynb",
+         "tokenize_texts": "30_text.core.ipynb",
+         "tokenize_df": "30_text.core.ipynb",
+         "tokenize_csv": "30_text.core.ipynb",
+         "load_tokenized_csv": "30_text.core.ipynb",
+         "Tokenizer": "30_text.core.ipynb",
+         "eu_langs": "30_text.core.ipynb",
+         "SentencePieceTokenizer": "30_text.core.ipynb",
+         "SubwordTokenizer": "30_text.core.ipynb",
+         "reverse_text": "31_text.data.ipynb",
+         "make_vocab": "31_text.data.ipynb",
+         "TensorText": "31_text.data.ipynb",
+         "LMTensorText": "31_text.data.ipynb",
+         "TensorText.__doc__": "31_text.data.ipynb",
+         "LMTensorText.__doc__": "31_text.data.ipynb",
+         "Numericalize": "31_text.data.ipynb",
+         "LMDataLoader": "31_text.data.ipynb",
+         "Pad_Input": "31_text.data.ipynb",
+         "pad_input": "31_text.data.ipynb",
+         "pad_chunk": "31_text.data.ipynb",
+         "pad_input_chunk": "31_text.data.ipynb",
+         "Pad_Chunk": "31_text.data.ipynb",
+         "SortedDL": "31_text.data.ipynb",
+         "TextBlock": "31_text.data.ipynb",
+         "TextDataLoaders": "31_text.data.ipynb",
+         "TextDataLoaders.from_csv": "31_text.data.ipynb",
+         "dropout_mask": "32_text.models.awdlstm.ipynb",
+         "RNNDropout": "32_text.models.awdlstm.ipynb",
+         "WeightDropout": "32_text.models.awdlstm.ipynb",
+         "EmbeddingDropout": "32_text.models.awdlstm.ipynb",
+         "AWD_LSTM": "32_text.models.awdlstm.ipynb",
+         "awd_lstm_lm_split": "32_text.models.awdlstm.ipynb",
+         "awd_lstm_lm_config": "32_text.models.awdlstm.ipynb",
+         "awd_lstm_clas_split": "32_text.models.awdlstm.ipynb",
+         "awd_lstm_clas_config": "32_text.models.awdlstm.ipynb",
+         "LinearDecoder": "33_text.models.core.ipynb",
+         "SequentialRNN": "33_text.models.core.ipynb",
+         "get_language_model": "33_text.models.core.ipynb",
+         "SentenceEncoder": "33_text.models.core.ipynb",
+         "masked_concat_pool": "33_text.models.core.ipynb",
+         "PoolingLinearClassifier": "33_text.models.core.ipynb",
+         "get_text_classifier": "33_text.models.core.ipynb",
+         "ModelResetter": "34_callback.rnn.ipynb",
+         "RNNCallback": "34_callback.rnn.ipynb",
+         "RNNRegularizer": "34_callback.rnn.ipynb",
+         "rnn_cbs": "34_callback.rnn.ipynb",
+         "match_embeds": "37_text.learner.ipynb",
+         "load_ignore_keys": "37_text.learner.ipynb",
+         "clean_raw_keys": "37_text.learner.ipynb",
+         "load_model_text": "37_text.learner.ipynb",
+         "TextLearner": "37_text.learner.ipynb",
+         "decode_spec_tokens": "37_text.learner.ipynb",
+         "LMLearner": "37_text.learner.ipynb",
+         "language_model_learner": "37_text.learner.ipynb",
+         "text_classifier_learner": "37_text.learner.ipynb",
+         "make_date": "40_tabular.core.ipynb",
+         "add_datepart": "40_tabular.core.ipynb",
+         "add_elapsed_times": "40_tabular.core.ipynb",
+         "cont_cat_split": "40_tabular.core.ipynb",
+         "df_shrink_dtypes": "40_tabular.core.ipynb",
+         "df_shrink": "40_tabular.core.ipynb",
+         "Tabular": "40_tabular.core.ipynb",
+         "TabularPandas": "40_tabular.core.ipynb",
+         "TabularProc": "40_tabular.core.ipynb",
+         "Categorify": "40_tabular.core.ipynb",
+         "setups": "40_tabular.core.ipynb",
+         "FillStrategy": "40_tabular.core.ipynb",
+         "FillMissing": "40_tabular.core.ipynb",
+         "ReadTabBatch": "40_tabular.core.ipynb",
+         "TabDataLoader": "40_tabular.core.ipynb",
+         "TabularDataLoaders": "41_tabular.data.ipynb",
+         "TabularDataLoaders.from_csv": "41_tabular.data.ipynb",
+         "emb_sz_rule": "42_tabular.model.ipynb",
+         "get_emb_sz": "42_tabular.model.ipynb",
+         "TabularModel": "42_tabular.model.ipynb",
+         "tabular_config": "42_tabular.model.ipynb",
+         "TabularLearner": "43_tabular.learner.ipynb",
+         "tabular_learner": "43_tabular.learner.ipynb",
+         "TabularCollab": "45_collab.ipynb",
+         "CollabDataLoaders": "45_collab.ipynb",
+         "CollabDataLoaders.from_csv": "45_collab.ipynb",
+         "EmbeddingDotBias": "45_collab.ipynb",
+         "EmbeddingNN": "45_collab.ipynb",
+         "collab_learner": "45_collab.ipynb",
+         "get_dicom_files": "60_medical.imaging.ipynb",
+         "Path.dcmread": "60_medical.imaging.ipynb",
+         "TensorDicom": "60_medical.imaging.ipynb",
+         "PILDicom": "60_medical.imaging.ipynb",
+         "Path.png16read": "60_medical.imaging.ipynb",
+         "pixels": "60_medical.imaging.ipynb",
+         "scaled_px": "60_medical.imaging.ipynb",
+         "array_freqhist_bins": "60_medical.imaging.ipynb",
+         "Tensor.freqhist_bins": "60_medical.imaging.ipynb",
+         "Tensor.hist_scaled_pt": "60_medical.imaging.ipynb",
+         "Tensor.hist_scaled": "60_medical.imaging.ipynb",
+         "DcmDataset.hist_scaled": "60_medical.imaging.ipynb",
+         "Tensor.windowed": "60_medical.imaging.ipynb",
+         "DcmDataset.windowed": "60_medical.imaging.ipynb",
+         "dicom_windows": "60_medical.imaging.ipynb",
+         "TensorCTScan": "60_medical.imaging.ipynb",
+         "PILCTScan": "60_medical.imaging.ipynb",
+         "DcmDataset.show": "60_medical.imaging.ipynb",
+         "DcmDataset.pct_in_window": "60_medical.imaging.ipynb",
+         "uniform_blur2d": "60_medical.imaging.ipynb",
+         "gauss_blur2d": "60_medical.imaging.ipynb",
+         "Tensor.mask_from_blur": "60_medical.imaging.ipynb",
+         "DcmDataset.mask_from_blur": "60_medical.imaging.ipynb",
+         "mask2bbox": "60_medical.imaging.ipynb",
+         "crop_resize": "60_medical.imaging.ipynb",
+         "Tensor.to_nchan": "60_medical.imaging.ipynb",
+         "DcmDataset.to_nchan": "60_medical.imaging.ipynb",
+         "Tensor.to_3chan": "60_medical.imaging.ipynb",
+         "DcmDataset.to_3chan": "60_medical.imaging.ipynb",
+         "Tensor.save_jpg": "60_medical.imaging.ipynb",
+         "DcmDataset.save_jpg": "60_medical.imaging.ipynb",
+         "Tensor.to_uint16": "60_medical.imaging.ipynb",
+         "DcmDataset.to_uint16": "60_medical.imaging.ipynb",
+         "Tensor.save_tif16": "60_medical.imaging.ipynb",
+         "DcmDataset.save_tif16": "60_medical.imaging.ipynb",
+         "DcmDataset.set_pixels": "60_medical.imaging.ipynb",
+         "DcmDataset.pixel_array": "60_medical.imaging.ipynb",
+         "DcmDataset.zoom": "60_medical.imaging.ipynb",
+         "DcmDataset.zoom_to": "60_medical.imaging.ipynb",
+         "DcmDataset.as_dict": "60_medical.imaging.ipynb",
+         "pd.DataFrame.from_dicoms": "60_medical.imaging.ipynb",
+         "DicomSegmentationDataLoaders": "60_medical.imaging.ipynb",
+         "WandbCallback": "70_callback.wandb.ipynb",
+         "Learner.gather_args": "70_callback.wandb.ipynb",
+         "log_dataset": "70_callback.wandb.ipynb",
+         "log_model": "70_callback.wandb.ipynb",
+         "TensorBoardBaseCallback": "70a_callback.tensorboard.ipynb",
+         "TensorBoardCallback": "70a_callback.tensorboard.ipynb",
+         "TensorBoardProjectorCallback": "70a_callback.tensorboard.ipynb",
+         "projector_word_embeddings": "70a_callback.tensorboard.ipynb",
+         "NeptuneCallback": "70b_callback.neptune.ipynb",
+         "json_clean": "70c_callback.captum.ipynb",
+         "jsonutil.json_clean": "70c_callback.captum.ipynb",
+         "CaptumInterpretation": "70c_callback.captum.ipynb",
+         "CaptumInterpretation.insights": "70c_callback.captum.ipynb",
+         "CometCallback": "70d_callback.comet.ipynb",
+         "synth_dbunch": "97_test_utils.ipynb",
+         "RegModel": "97_test_utils.ipynb",
+         "synth_learner": "97_test_utils.ipynb",
+         "VerboseCallback": "97_test_utils.ipynb",
+         "get_env": "97_test_utils.ipynb",
+         "try_import": "97_test_utils.ipynb",
+         "nvidia_smi": "97_test_utils.ipynb",
+         "nvidia_mem": "97_test_utils.ipynb",
+         "show_install": "97_test_utils.ipynb",
+         "PYTORCH_URL": "99_pytorch_doc.ipynb",
+         "pytorch_doc_link": "99_pytorch_doc.ipynb"}
+
+modules = ["torch_core.py",
+           "layers.py",
+           "losses.py",
+           "data/load.py",
+           "data/core.py",
+           "data/external.py",
+           "data/transforms.py",
+           "data/block.py",
+           "vision/core.py",
+           "vision/data.py",
+           "vision/augment.py",
+           "vision/utils.py",
+           "vision/widgets.py",
+           "vision/models/xresnet.py",
+           "optimizer.py",
+           "callback/core.py",
+           "learner.py",
+           "metrics.py",
+           "callback/schedule.py",
+           "callback/data.py",
+           "callback/hook.py",
+           "vision/models/unet.py",
+           "callback/progress.py",
+           "callback/tracker.py",
+           "callback/fp16.py",
+           "callback/training.py",
+           "callback/preds.py",
+           "callback/mixup.py",
+           "interpret.py",
+           "distributed.py",
+           "vision/learner.py",
+           "vision/gan.py",
+           "text/core.py",
+           "text/data.py",
+           "text/models/awdlstm.py",
+           "text/models/core.py",
+           "callback/rnn.py",
+           "text/learner.py",
+           "tabular/core.py",
+           "tabular/data.py",
+           "tabular/model.py",
+           "tabular/learner.py",
+           "collab.py",
+           "medical/imaging.py",
+           "medical/text.py",
+           "callback/wandb.py",
+           "callback/tensorboard.py",
+           "callback/neptune.py",
+           "callback/captum.py",
+           "callback/comet.py",
+           "test_utils.py",
+           "_pytorch_doc.py"]
+
+doc_url = "https://docs.fast.ai/"
+
+git_url = "https://github.com/fastai/fastai/tree/master/"
+
+def custom_doc_links(name):
+    from nbdev.showdoc import try_external_doc_link
+    return try_external_doc_link(name, ['fastcore', 'nbdev'])
+

fastai/_pytorch_doc.py

@@ -0,0 +1,46 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/99_pytorch_doc.ipynb.
+
+# %% ../nbs/99_pytorch_doc.ipynb 5
+from __future__ import annotations
+from types import ModuleType
+
+# %% auto 0
+__all__ = ['PYTORCH_URL', 'pytorch_doc_link']
+
+# %% ../nbs/99_pytorch_doc.ipynb 6
+PYTORCH_URL = 'https://pytorch.org/docs/stable/'
+
+# %% ../nbs/99_pytorch_doc.ipynb 7
+def _mod2page(
+    mod:ModuleType, # A PyTorch module
+) -> str:
+    "Get the webpage name for a PyTorch module"
+    if mod == Tensor: return 'tensors.html'
+    name = mod.__name__
+    name = name.replace('torch.', '').replace('utils.', '')
+    if name.startswith('nn.modules'): return 'nn.html'
+    return f'{name}.html'
+
+# %% ../nbs/99_pytorch_doc.ipynb 9
+import importlib
+
+# %% ../nbs/99_pytorch_doc.ipynb 10
+def pytorch_doc_link(
+    name:str # Name of a PyTorch module, class or function
+) -> (str, None):
+    "Get the URL to the documentation of a PyTorch module, class or function"
+    if name.startswith('F'): name = 'torch.nn.functional' + name[1:]
+    if not name.startswith('torch.'): name = 'torch.' + name
+    if name == 'torch.Tensor': return f'{PYTORCH_URL}tensors.html'
+    try:
+        mod = importlib.import_module(name)
+        return f'{PYTORCH_URL}{_mod2page(mod)}'
+    except: pass
+    splits = name.split('.')
+    mod_name,fname = '.'.join(splits[:-1]),splits[-1]
+    if mod_name == 'torch.Tensor': return f'{PYTORCH_URL}tensors.html#{name}'
+    try:
+        mod = importlib.import_module(mod_name)
+        page = _mod2page(mod)
+        return f'{PYTORCH_URL}{page}#{name}'
+    except: return None

fastai/basics.py

@@ -0,0 +1,6 @@
+from .data.all import *
+from .optimizer import *
+from .callback.core import *
+from .learner import *
+from .metrics import *
+from .interpret import *

fastai/callback/__init__.py

@@ -0,0 +1 @@
+

fastai/callback/all.py

@@ -0,0 +1,12 @@
+from .core import *
+from .data import *
+from .fp16 import *
+from .hook import *
+from .mixup import *
+from .progress import *
+from .schedule import *
+from .tracker import *
+from .rnn import *
+from .training import *
+from .preds import *
+from .channelslast import *

fastai/callback/azureml.py

@@ -0,0 +1,72 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/74_callback.azureml.ipynb (unless otherwise specified).
+
+__all__ = ['AzureMLCallback']
+
+# Cell
+from ..basics import *
+from ..learner import Callback
+
+# Cell
+from azureml.core.run import Run
+from azureml.exceptions import RunEnvironmentException
+import warnings
+
+# Cell
+class AzureMLCallback(Callback):
+    """
+    Log losses, metrics, model architecture summary to AzureML.
+
+    If `log_offline` is False, will only log if actually running on AzureML.
+    A custom AzureML `Run` class can be passed as `azurerun`.
+    If `log_to_parent` is True, will also log to the parent run, if exists (e.g. in AzureML pipelines).
+    """
+    order = Recorder.order+1
+
+    def __init__(self, azurerun=None, log_to_parent=True):
+        if azurerun:
+            self.azurerun = azurerun
+        else:
+            try:
+                self.azurerun = Run.get_context(allow_offline=False)
+            except RunEnvironmentException:
+                # running locally
+                self.azurerun = None
+                warnings.warn("Not running on AzureML and no azurerun passed, AzureMLCallback will be disabled.")
+        self.log_to_parent = log_to_parent
+
+    def before_fit(self):
+        self._log("n_epoch", self.learn.n_epoch)
+        self._log("model_class", str(type(self.learn.model)))
+
+        try:
+            summary_file = Path("outputs") / 'model_summary.txt'
+            with summary_file.open("w") as f:
+                f.write(repr(self.learn.model))
+        except:
+            print('Did not log model summary. Check if your model is PyTorch model.')
+
+    def after_batch(self):
+        # log loss and opt.hypers
+        if self.learn.training:
+            self._log('batch__loss', self.learn.loss.item())
+            self._log('batch__train_iter', self.learn.train_iter)
+            for i, h in enumerate(self.learn.opt.hypers):
+                for k, v in h.items():
+                    self._log(f'batch__opt.hypers.{k}', v)
+
+    def after_epoch(self):
+        # log metrics
+        for n, v in zip(self.learn.recorder.metric_names, self.learn.recorder.log):
+            if n not in ['epoch', 'time']:
+                self._log(f'epoch__{n}', v)
+            if n == 'time':
+                # split elapsed time string, then convert into 'seconds' to log
+                m, s = str(v).split(':')
+                elapsed = int(m)*60 + int(s)
+                self._log(f'epoch__{n}', elapsed)
+
+    def _log(self, metric, value):
+        if self.azurerun is not None:
+            self.azurerun.log(metric, value)
+            if self.log_to_parent and self.azurerun.parent is not None:
+                self.azurerun.parent.log(metric, value)

fastai/callback/captum.py

@@ -0,0 +1,113 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/70c_callback.captum.ipynb.
+
+# %% ../../nbs/70c_callback.captum.ipynb 3
+from __future__ import annotations
+import tempfile
+from ..basics import *
+
+# %% auto 0
+__all__ = ['CaptumInterpretation']
+
+# %% ../../nbs/70c_callback.captum.ipynb 6
+from ipykernel import jsonutil
+
+# %% ../../nbs/70c_callback.captum.ipynb 7
+# Dirty hack as json_clean doesn't support CategoryMap type
+_json_clean=jsonutil.json_clean
+def json_clean(o):
+    o = list(o.items) if isinstance(o,CategoryMap) else o
+    return _json_clean(o)
+
+jsonutil.json_clean = json_clean
+
+# %% ../../nbs/70c_callback.captum.ipynb 8
+from captum.attr import IntegratedGradients,NoiseTunnel,GradientShap,Occlusion
+from captum.attr import visualization as viz
+
+from matplotlib.colors import LinearSegmentedColormap
+
+from captum.insights import AttributionVisualizer, Batch
+from captum.insights.attr_vis.features import ImageFeature
+
+# %% ../../nbs/70c_callback.captum.ipynb 16
+class CaptumInterpretation():
+    "Captum Interpretation for Resnet"
+    def __init__(self,learn,cmap_name='custom blue',colors=None,N=256,methods=('original_image','heat_map'),
+                 signs=("all", "positive"),outlier_perc=1):
+        if colors is None: colors = [(0, '#ffffff'),(0.25, '#000000'),(1, '#000000')]
+        store_attr()
+        self.dls,self.model = learn.dls,self.learn.model
+        self.supported_metrics=['IG','NT','Occl']
+
+    def get_baseline_img(self, img_tensor,baseline_type):
+        baseline_img=None
+        if baseline_type=='zeros': baseline_img= img_tensor*0
+        if baseline_type=='uniform': baseline_img= torch.rand(img_tensor.shape)
+        if baseline_type=='gauss':
+            baseline_img= (torch.rand(img_tensor.shape).to(self.dls.device)+img_tensor)/2
+        return baseline_img.to(self.dls.device)
+
+    def visualize(self,inp,metric='IG',n_steps=1000,baseline_type='zeros',nt_type='smoothgrad', strides=(3,4,4), sliding_window_shapes=(3,15,15)):
+        if metric not in self.supported_metrics:
+            raise Exception(f"Metric {metric} is not supported. Currently {self.supported_metrics} are only supported")
+        tls = L([TfmdLists(inp, t) for t in L(ifnone(self.dls.tfms,[None]))])
+        inp_data=list(zip(*(tls[0],tls[1])))[0]
+        enc_data,dec_data=self._get_enc_dec_data(inp_data)
+        attributions=self._get_attributions(enc_data,metric,n_steps,nt_type,baseline_type,strides,sliding_window_shapes)
+        self._viz(attributions,dec_data,metric)
+
+    def _viz(self,attributions,dec_data,metric):
+        default_cmap = LinearSegmentedColormap.from_list(self.cmap_name,self.colors, N=self.N)
+        _ = viz.visualize_image_attr_multiple(np.transpose(attributions.squeeze().cpu().detach().numpy(), (1,2,0)),
+                                              np.transpose(dec_data[0].numpy(), (1,2,0)),
+                                              methods=self.methods,
+                                              cmap=default_cmap,
+                                              show_colorbar=True,
+                                              signs=self.signs,
+                                              outlier_perc=self.outlier_perc, titles=[f'Original Image - ({dec_data[1]})', metric])
+
+
+
+    def _get_enc_dec_data(self,inp_data):
+        dec_data=self.dls.after_item(inp_data)
+        enc_data=self.dls.after_batch(to_device(self.dls.before_batch(dec_data),self.dls.device))
+        return(enc_data,dec_data)
+
+    def _get_attributions(self,enc_data,metric,n_steps,nt_type,baseline_type,strides,sliding_window_shapes):
+        # Get Baseline
+        baseline=self.get_baseline_img(enc_data[0],baseline_type)
+        supported_metrics ={}
+        if metric == 'IG':
+            self._int_grads = self._int_grads if hasattr(self,'_int_grads') else IntegratedGradients(self.model)
+            return self._int_grads.attribute(enc_data[0],baseline, target=enc_data[1], n_steps=200)
+        elif metric == 'NT':
+            self._int_grads = self._int_grads if hasattr(self,'_int_grads') else IntegratedGradients(self.model)
+            self._noise_tunnel= self._noise_tunnel if hasattr(self,'_noise_tunnel') else NoiseTunnel(self._int_grads)
+            return self._noise_tunnel.attribute(enc_data[0].to(self.dls.device), n_samples=1, nt_type=nt_type, target=enc_data[1])
+        elif metric == 'Occl':
+            self._occlusion = self._occlusion if hasattr(self,'_occlusion') else Occlusion(self.model)
+            return self._occlusion.attribute(enc_data[0].to(self.dls.device),
+                                       strides = strides,
+                                       target=enc_data[1],
+                                       sliding_window_shapes=sliding_window_shapes,
+                                       baselines=baseline)
+
+# %% ../../nbs/70c_callback.captum.ipynb 26
+@patch
+def insights(x: CaptumInterpretation,inp_data,debug=True):
+    _baseline_func= lambda o: o*0
+    _get_vocab = lambda vocab: list(map(str,vocab)) if isinstance(vocab[0],bool) else vocab
+    dl = x.dls.test_dl(L(inp_data),with_labels=True, bs=4)
+    normalize_func= next((func for func in dl.after_batch if type(func)==Normalize),noop)
+
+    # captum v0.3 expects tensors without the batch dimension.
+    if nested_attr(normalize_func, 'mean.ndim', 4)==4: normalize_func.mean.squeeze_(0) 
+    if nested_attr(normalize_func, 'std.ndim', 4)==4: normalize_func.std.squeeze_(0) 
+
+    visualizer = AttributionVisualizer(
+        models=[x.model],
+        score_func=lambda o: torch.nn.functional.softmax(o, 1),
+        classes=_get_vocab(dl.vocab),
+        features=[ImageFeature("Image", baseline_transforms=[_baseline_func], input_transforms=[normalize_func])],
+        dataset=x._formatted_data_iter(dl,normalize_func))
+    visualizer.render(debug=debug)

fastai/callback/channelslast.py

@@ -0,0 +1,41 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/18c_callback.channelslast.ipynb.
+
+# %% ../../nbs/18c_callback.channelslast.ipynb 1
+from __future__ import annotations
+from ..basics import *
+from .fp16 import MixedPrecision
+
+from torch.cuda.amp import GradScaler
+
+# %% auto 0
+__all__ = ['ChannelsLast']
+
+# %% ../../nbs/18c_callback.channelslast.ipynb 7
+class ChannelsLast(Callback):
+    "Channels last training using PyTorch's Channels Last Memory Format (beta)"
+    order = -1 # Needs to run before any model modification callbacks occur
+    def before_fit(self):
+        self.learn.model.to(memory_format=torch.channels_last)
+
+# %% ../../nbs/18c_callback.channelslast.ipynb 9
+@patch
+@delegates(GradScaler)
+def to_channelslast(self:Learner,
+    to_fp16:bool=True, # Add `MixedPrecision` callback. Recommended for full channels last performance
+    **kwargs
+):
+    "Set `Learner` and inputs to `channels_last` format and `MixedPrecision` by default"
+    if to_fp16 and not hasattr(self, 'mixed_precision') and not hasattr(self, 'channels_last'):
+        return self.add_cbs([ChannelsLast(), MixedPrecision(**kwargs)])
+    elif not hasattr(self, 'channels_last'):
+        return self.add_cb(ChannelsLast())
+
+# %% ../../nbs/18c_callback.channelslast.ipynb 10
+@patch
+def to_contiguous(self:Learner, to_fp32:bool=False):
+    "Set `Learner` and inputs to `contiguous_format` (default format), optionally to single precision"
+    self.model.to(memory_format=torch.contiguous_format)
+    if to_fp32:
+        return self.remove_cbs([ChannelsLast, MixedPrecision])
+    else:
+        return self.remove_cb(ChannelsLast)

fastai/callback/comet.py

@@ -0,0 +1,91 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/70d_callback.comet.ipynb.
+
+# %% ../../nbs/70d_callback.comet.ipynb 3
+from __future__ import annotations
+
+import tempfile
+
+from ..basics import *
+from ..learner import Callback
+
+# %% auto 0
+__all__ = ['CometCallback']
+
+# %% ../../nbs/70d_callback.comet.ipynb 12
+import comet_ml
+
+# %% ../../nbs/70d_callback.comet.ipynb 13
+class CometCallback(Callback):
+    "Log losses, metrics, model weights, model architecture summary to neptune"
+    order = Recorder.order + 1
+
+    def __init__(self, project_name, log_model_weights=True):
+        self.log_model_weights = log_model_weights
+        self.keep_experiment_running = keep_experiment_running
+        self.project_name = project_name
+        self.experiment = None
+
+    def before_fit(self):
+        try:
+            self.experiment = comet_ml.Experiment(project_name=self.project_name)
+        except ValueError:
+            print("No active experiment")
+
+        try:
+            self.experiment.log_parameter("n_epoch", str(self.learn.n_epoch))
+            self.experiment.log_parameter("model_class", str(type(self.learn.model)))
+        except:
+            print(f"Did not log all properties.")
+
+        try:
+            with tempfile.NamedTemporaryFile(mode="w") as f:
+                with open(f.name, "w") as g:
+                    g.write(repr(self.learn.model))
+                self.experiment.log_asset(f.name, "model_summary.txt")
+        except:
+            print("Did not log model summary. Check if your model is PyTorch model.")
+
+        if self.log_model_weights and not hasattr(self.learn, "save_model"):
+            print(
+                "Unable to log model to Comet.\n",
+            )
+
+    def after_batch(self):
+        # log loss and opt.hypers
+        if self.learn.training:
+            self.experiment.log_metric("batch__smooth_loss", self.learn.smooth_loss)
+            self.experiment.log_metric("batch__loss", self.learn.loss)
+            self.experiment.log_metric("batch__train_iter", self.learn.train_iter)
+            for i, h in enumerate(self.learn.opt.hypers):
+                for k, v in h.items():
+                    self.experiment.log_metric(f"batch__opt.hypers.{k}", v)
+
+    def after_epoch(self):
+        # log metrics
+        for n, v in zip(self.learn.recorder.metric_names, self.learn.recorder.log):
+            if n not in ["epoch", "time"]:
+                self.experiment.log_metric(f"epoch__{n}", v)
+            if n == "time":
+                self.experiment.log_text(f"epoch__{n}", str(v))
+
+        # log model weights
+        if self.log_model_weights and hasattr(self.learn, "save_model"):
+            if self.learn.save_model.every_epoch:
+                _file = join_path_file(
+                    f"{self.learn.save_model.fname}_{self.learn.save_model.epoch}",
+                    self.learn.path / self.learn.model_dir,
+                    ext=".pth",
+                )
+            else:
+                _file = join_path_file(
+                    self.learn.save_model.fname,
+                    self.learn.path / self.learn.model_dir,
+                    ext=".pth",
+                )
+            self.experiment.log_asset(_file)
+
+    def after_fit(self):
+        try:
+            self.experiment.end()
+        except:
+            print("No neptune experiment to stop.")

fastai/callback/core.py

@@ -0,0 +1,187 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/13_callback.core.ipynb.
+
+# %% ../../nbs/13_callback.core.ipynb 2
+from __future__ import annotations
+from ..data.all import *
+from ..optimizer import *
+from ..losses import BaseLoss
+
+# %% auto 0
+__all__ = ['Callback', 'TrainEvalCallback', 'GatherPredsCallback', 'FetchPredsCallback', 'CancelStepException',
+           'CancelBackwardException', 'CancelFitException', 'CancelEpochException', 'CancelTrainException',
+           'CancelValidException', 'CancelBatchException', 'event']
+
+# %% ../../nbs/13_callback.core.ipynb 4
+_all_ = ['CancelStepException','CancelBackwardException','CancelFitException','CancelEpochException','CancelTrainException','CancelValidException','CancelBatchException']
+
+# %% ../../nbs/13_callback.core.ipynb 8
+_events = L.split('after_create before_fit before_epoch before_train before_batch after_pred after_loss \
+    before_backward after_cancel_backward after_backward before_step after_cancel_step after_step \
+    after_cancel_batch after_batch after_cancel_train after_train before_validate after_cancel_validate \
+    after_validate after_cancel_epoch after_epoch after_cancel_fit after_fit')
+
+mk_class('event', **_events.map_dict(),
+         doc="All possible events as attributes to get tab-completion and typo-proofing")
+
+# %% ../../nbs/13_callback.core.ipynb 9
+_all_ = ['event']
+
+# %% ../../nbs/13_callback.core.ipynb 14
+_inner_loop = "before_batch after_pred after_loss before_backward after_cancel_backward after_backward before_step after_step after_cancel_batch after_batch".split()
+
+# %% ../../nbs/13_callback.core.ipynb 15
+_ex_docs = dict(
+    CancelBatchException="Skip the rest of this batch and go to `after_batch`",
+    CancelTrainException="Skip the rest of the training part of the epoch and go to `after_train`",
+    CancelValidException="Skip the rest of the validation part of the epoch and go to `after_validate`",
+    CancelEpochException="Skip the rest of this epoch and go to `after_epoch`",
+    CancelStepException ="Skip stepping the optimizer",
+    CancelBackwardException="Skip the backward pass and go to `after_backward`",
+    CancelFitException  ="Interrupts training and go to `after_fit`")
+
+for c,d in _ex_docs.items(): mk_class(c,sup=Exception,doc=d)
+
+# %% ../../nbs/13_callback.core.ipynb 16
+@funcs_kwargs(as_method=True)
+class Callback(Stateful,GetAttr):
+    "Basic class handling tweaks of the training loop by changing a `Learner` in various events"
+    order,_default,learn,run,run_train,run_valid = 0,'learn',None,True,True,True
+    _methods = _events
+
+    def __init__(self, **kwargs): assert not kwargs, f'Passed unknown events: {kwargs}'
+    def __repr__(self): return type(self).__name__
+
+    def __call__(self, event_name):
+        "Call `self.{event_name}` if it's defined"
+        _run = (event_name not in _inner_loop or (self.run_train and getattr(self, 'training', True)) or
+               (self.run_valid and not getattr(self, 'training', False)))
+        res = None
+        if self.run and _run: 
+            try: res = getcallable(self, event_name)()
+            except (CancelBatchException, CancelBackwardException, CancelEpochException, CancelFitException, CancelStepException, CancelTrainException, CancelValidException): raise
+            except Exception as e: raise modify_exception(e, f'Exception occured in `{self.__class__.__name__}` when calling event `{event_name}`:\n\t{e.args[0]}', replace=True)
+        if event_name=='after_fit': self.run=True #Reset self.run to True at each end of fit
+        return res
+
+    def __setattr__(self, name, value):
+        "Set an attribute for a `Callback`"
+        if hasattr(self.learn,name):
+            warn(f"You are shadowing an attribute ({name}) that exists in the learner. Use `self.learn.{name}` to avoid this")
+        super().__setattr__(name, value)
+
+    @property
+    def name(self):
+        "Name of the `Callback`, camel-cased and with '*Callback*' removed"
+        return class2attr(self, 'Callback')
+
+# %% ../../nbs/13_callback.core.ipynb 34
+class TrainEvalCallback(Callback):
+    "`Callback` that tracks the number of iterations done and properly sets training/eval mode"
+    order,run_valid = -10,False
+    def after_create(self): self.learn.n_epoch = 1
+
+    def before_fit(self):
+        "Set the iter and epoch counters to 0, put the model and the right device"
+        self.learn.epoch,self.learn.loss = 0,tensor(0.)
+        self.learn.train_iter,self.learn.pct_train = 0,0.
+        device = getattr(self.dls, 'device', default_device())
+        self.model.to(device)
+        if isinstance(self.loss_func, (nn.Module, BaseLoss)): self.loss_func.to(device)
+        if hasattr(self.model, 'reset'): self.model.reset()
+
+    def after_batch(self):
+        "Update the iter counter (in training mode)"
+        self.learn.pct_train += 1./(self.n_iter*self.n_epoch)
+        self.learn.train_iter += 1
+
+    def before_train(self):
+        "Set the model to training mode"
+        self.learn.pct_train=self.epoch/self.n_epoch
+        self.model.train()
+        self.learn.training=True
+
+    def before_validate(self):
+        "Set the model to validation mode"
+        self.model.eval()
+        self.learn.training=False
+
+# %% ../../nbs/13_callback.core.ipynb 38
+if not hasattr(defaults, 'callbacks'): defaults.callbacks = [TrainEvalCallback]
+
+# %% ../../nbs/13_callback.core.ipynb 52
+class GatherPredsCallback(Callback):
+    "`Callback` that returns all predictions and targets, optionally `with_input` or `with_loss`"
+    _stateattrs=('preds','targets','inputs','losses')
+    def __init__(self,
+        with_input:bool=False, # Whether to return inputs
+        with_loss:bool=False, # Whether to return losses
+        save_preds:Path=None, # Path to save predictions
+        save_targs:Path=None, # Path to save targets
+        with_preds:bool=True, # Whether to return predictions
+        with_targs:bool=True, # Whether to return targets
+        concat_dim:int=0, # Dimension to concatenate returned tensors
+        pickle_protocol:int=2 # Pickle protocol used to save predictions and targets
+    ):
+        store_attr()
+
+    def before_batch(self):
+        "If `with_input`, detach batch inputs"
+        if self.with_input: self.inputs.append((self.learn.to_detach(self.xb)))
+
+    def before_validate(self):
+        "Initialize containers"
+        self.preds,self.targets = [],[]
+        if self.with_input: self.inputs = []
+        if self.with_loss:  self.losses = []
+
+    def after_batch(self):
+        "Save predictions, targets and potentially losses"
+        if not hasattr(self, 'pred'): return
+        preds,targs = self.learn.to_detach(self.pred),self.learn.to_detach(self.yb)
+        if self.with_preds: self.preds.append(preds)
+        if self.with_targs: self.targets.append(targs)
+        if self.save_preds is not None: 
+            torch.save(preds, self.save_preds/str(self.iter), pickle_protocol=self.pickle_protocol)
+        if self.save_targs is not None: 
+            torch.save(targs[0], self.save_targs/str(self.iter), pickle_protocol=self.pickle_protocol)
+        if self.with_loss:
+            bs = find_bs(self.yb)
+            loss = self.loss if self.loss.numel() == bs else self.loss.view(bs,-1).mean(1)
+            self.losses.append(self.learn.to_detach(loss))
+
+    def after_validate(self):
+        "Concatenate all recorded tensors"
+        if not hasattr(self, 'preds'): return
+        if self.with_input: self.inputs  = detuplify(to_concat(self.inputs, dim=self.concat_dim))
+        if self.with_preds: self.preds   = detuplify(to_concat(self.preds, dim=self.concat_dim))
+        if self.with_targs: self.targets = detuplify(to_concat(self.targets, dim=self.concat_dim))
+        if self.with_loss:  self.losses  = to_concat(self.losses)
+
+    def all_tensors(self) -> (Tensor, list):
+        "Returns all recorded tensors in the order [inputs, preds, targets, losses]"
+        res = [self.preds if self.with_preds else None, self.targets if self.with_targs else None]
+        if self.with_input: res = [self.inputs] + res
+        if self.with_loss:  res.append(self.losses)
+        return res
+
+# %% ../../nbs/13_callback.core.ipynb 54
+class FetchPredsCallback(Callback):
+    "A callback to fetch predictions during the training loop"
+    remove_on_fetch = True
+    def __init__(self,
+        ds_idx:int=1, # Index of dataset, 0 for train, 1 for valid, used if `dl` is not present
+        dl:DataLoader=None, # `DataLoader` used for fetching `Learner` predictions
+        with_input:bool=False, # Whether to return inputs in `GatherPredsCallback`
+        with_decoded:bool=False, # Whether to return decoded predictions
+        cbs:Callback|MutableSequence=None, # `Callback` to temporarily remove from `Learner`
+        reorder:bool=True # Whether to sort prediction results
+    ):
+        self.cbs = L(cbs)
+        store_attr('ds_idx,dl,with_input,with_decoded,reorder')
+
+    def after_validate(self):
+        "Fetch predictions from `Learner` without `self.cbs` and `remove_on_fetch` callbacks"
+        to_rm = L(cb for cb in self.learn.cbs if getattr(cb, 'remove_on_fetch', False))
+        with self.learn.removed_cbs(to_rm + self.cbs) as learn:
+            self.preds = learn.get_preds(ds_idx=self.ds_idx, dl=self.dl,
+                with_input=self.with_input, with_decoded=self.with_decoded, inner=True, reorder=self.reorder)

fastai/callback/data.py

@@ -0,0 +1,71 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/14a_callback.data.ipynb.
+
+# %% ../../nbs/14a_callback.data.ipynb 3
+from __future__ import annotations
+from ..basics import *
+
+# %% auto 0
+__all__ = ['CollectDataCallback', 'WeightedDL', 'PartialDL']
+
+# %% ../../nbs/14a_callback.data.ipynb 5
+class CollectDataCallback(Callback):
+    "Collect all batches, along with `pred` and `loss`, into `self.data`. Mainly for testing"
+    def before_fit(self): self.data = L()
+    def after_batch(self): 
+        self.data.append(self.learn.to_detach((self.xb,self.yb,self.pred,self.loss)))
+
+# %% ../../nbs/14a_callback.data.ipynb 6
+@delegates()
+class WeightedDL(TfmdDL):
+    "Weighted dataloader where `wgts` is used for the training set only"
+    def __init__(self, dataset=None, bs=None, wgts=None, **kwargs):
+        wgts = array([1.]*len(dataset) if wgts is None else wgts)
+        self.wgts = wgts/wgts.sum()
+        super().__init__(dataset=dataset, bs=bs, **kwargs)
+
+    def get_idxs(self):
+        if self.n==0: return []
+        if not self.shuffle: return super().get_idxs()
+        return list(np.random.choice(self.n, self.n, p=self.wgts))
+
+# %% ../../nbs/14a_callback.data.ipynb 7
+@patch
+@delegates(Datasets.dataloaders)
+def weighted_dataloaders(self:Datasets, wgts, bs=64, **kwargs):
+    "Create a weighted dataloader `WeightedDL` with `wgts` for the training set"
+    xtra_kwargs = [{}] * (self.n_subsets-1)
+    return self.dataloaders(bs=bs, dl_type=WeightedDL, dl_kwargs=({'wgts':wgts}, *xtra_kwargs), **kwargs)
+
+# %% ../../nbs/14a_callback.data.ipynb 12
+@patch
+@delegates(Datasets.weighted_dataloaders)
+def weighted_dataloaders(self:DataBlock, source, wgts, bs=64, verbose:bool=False, **kwargs):
+    "Create a weighted dataloader `WeightedDL` with `wgts` for the dataset"
+    dss = self.datasets(source, verbose=verbose)
+    if not hasattr(wgts, '__array__'): wgts = np.array(wgts)
+    trn_wgts = wgts[dss.splits[0]]
+    return dss.weighted_dataloaders(trn_wgts, bs=bs, after_batch=self.batch_tfms, after_item=self.item_tfms, **kwargs)
+
+# %% ../../nbs/14a_callback.data.ipynb 14
+@delegates()
+class PartialDL(TfmdDL):
+    "Select randomly partial quantity of data at each epoch"
+    def __init__(self, dataset=None, bs=None, partial_n=None, **kwargs):
+        super().__init__(dataset=dataset, bs=bs, **kwargs)
+        self.partial_n = min(partial_n, self.n) if partial_n else None
+
+    def get_idxs(self):
+        if self.partial_n is None: return super().get_idxs()
+        return list(np.random.choice(self.n, self.partial_n, replace=False))
+
+    def __len__(self):
+        if self.partial_n is None: return super().__len__()
+        return self.partial_n//self.bs + (0 if self.drop_last or self.partial_n%self.bs==0 else 1)
+
+# %% ../../nbs/14a_callback.data.ipynb 15
+@patch
+@delegates(Datasets.dataloaders)
+def partial_dataloaders(self:FilteredBase, partial_n, bs=64, **kwargs):
+    "Create a partial dataloader `PartialDL` for the training set"
+    xtra_kwargs = [{}] * (self.n_subsets-1)
+    return self.dataloaders(bs=bs, dl_type=PartialDL, dl_kwargs=({'partial_n':partial_n}, *xtra_kwargs), **kwargs)

fastai/callback/fp16.py

@@ -0,0 +1,217 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/18_callback.fp16.ipynb.
+
+# %% ../../nbs/18_callback.fp16.ipynb 2
+from __future__ import annotations
+from ..basics import *
+from .progress import *
+
+from torch.cuda.amp import GradScaler,autocast
+from torch.cuda.amp.grad_scaler import OptState
+
+# %% auto 0
+__all__ = ['MixedPrecision', 'FP16TestCallback', 'get_master', 'to_master_grads', 'to_model_params', 'test_overflow',
+           'grad_overflow', 'copy_clone', 'ModelToHalf', 'NonNativeMixedPrecision']
+
+# %% ../../nbs/18_callback.fp16.ipynb 17
+@delegates(GradScaler)
+class MixedPrecision(Callback):
+    "Mixed precision training using Pytorch's `autocast` and `GradScaler`"
+    order = 10
+    def __init__(self, **kwargs): self.kwargs = kwargs
+    def before_fit(self): 
+        self.autocast,self.learn.scaler,self.scales = autocast(),GradScaler(**self.kwargs),L()
+    def before_batch(self): self.autocast.__enter__()
+    def after_pred(self):
+        if next(flatten(self.pred)).dtype==torch.float16: self.learn.pred = to_float(self.pred)
+    def after_loss(self): self.autocast.__exit__(None, None, None)
+    def before_backward(self): self.learn.loss_grad = self.scaler.scale(self.loss_grad)
+    def before_step(self):
+        "Use `self` as a fake optimizer. `self.skipped` will be set to True `after_step` if gradients overflow. "
+        self.skipped=True
+        self.scaler.step(self)
+        if self.skipped: raise CancelStepException()
+        self.scales.append(self.scaler.get_scale())
+    def after_step(self): self.learn.scaler.update()
+
+    @property 
+    def param_groups(self): 
+        "Pretend to be an optimizer for `GradScaler`"
+        return self.opt.param_groups
+    def step(self, *args, **kwargs): 
+        "Fake optimizer step to detect whether this batch was skipped from `GradScaler`"
+        self.skipped=False
+    def after_fit(self): self.autocast,self.learn.scaler,self.scales = None,None,None
+
+# %% ../../nbs/18_callback.fp16.ipynb 19
+class FP16TestCallback(Callback):
+    "Asserts that predictions are `float16` values"
+    order = 9
+    def after_pred(self): assert listify(flatten(self.pred))[0].dtype==torch.float16
+
+# %% ../../nbs/18_callback.fp16.ipynb 22
+@patch
+@delegates(GradScaler)
+def to_fp16(self:Learner, **kwargs): return self.add_cb(MixedPrecision(**kwargs))
+
+# %% ../../nbs/18_callback.fp16.ipynb 23
+@patch
+def to_fp32(self:Learner): return self.remove_cb(MixedPrecision)
+
+# %% ../../nbs/18_callback.fp16.ipynb 26
+from ..fp16_utils import convert_network, model_grads_to_master_grads, master_params_to_model_params
+
+# %% ../../nbs/18_callback.fp16.ipynb 32
+from torch.nn.utils import parameters_to_vector
+
+# %% ../../nbs/18_callback.fp16.ipynb 33
+def get_master(
+    opt:Optimizer, # Optimizer from which to retrieve model params
+    flat_master:bool=False, # Flatten fp32 params into a vector for better performance
+) -> list: # List of fp16 params, and list of fp32 params
+    "Creates fp16 model params given an initialized `Optimizer`, also returning fp32 model params. "
+    model_params = [[param for param in pg if getattr(param, 'requires_grad', False) and hasattr(param, 'data')] for pg in opt.param_lists]
+    if flat_master:
+        master_params = []
+        for pg in model_params:
+            mp = parameters_to_vector([param.data.float() for param in pg])
+            mp = nn.Parameter(mp, requires_grad=True)
+            if mp.grad is None: mp.grad = mp.new(*mp.size())
+            master_params.append([mp])
+    else:
+        master_params = [[nn.Parameter(param.data.clone().float().detach(), requires_grad=True) for param in pg] for pg in model_params]
+    return model_params, master_params
+
+# %% ../../nbs/18_callback.fp16.ipynb 38
+def to_master_grads( 
+    model_pgs:list, # Fp16 model parameters to copy gradients from
+    master_pgs:list, # Fp32 model parameters to copy gradients to
+    flat_master:bool=False, # Whether or not fp32 parameters were previously flattened
+):
+    "Move fp16 model gradients to fp32 master gradients"
+    for (model_params,master_params) in zip(model_pgs,master_pgs):
+        model_grads_to_master_grads(model_params, master_params, flat_master=flat_master)
+
+# %% ../../nbs/18_callback.fp16.ipynb 42
+def to_model_params(
+    model_pgs:list, # Fp16 model params to copy to
+    master_pgs:list, # Fp32 master params to copy from
+    flat_master:bool=False # Whether master_pgs was previously flattened
+)->None:
+    "Copy updated fp32 master params to fp16 model params after gradient step. " 
+    for (model_params,master_params) in zip(model_pgs,master_pgs):
+        master_params_to_model_params(model_params, master_params, flat_master=flat_master)
+
+# %% ../../nbs/18_callback.fp16.ipynb 47
+def test_overflow(x:torch.Tensor):
+    "Tests whether fp16 gradients have overflown."
+    s = float(x.float().sum())
+    return (s == float('inf') or s == float('-inf') or s != s)
+
+# %% ../../nbs/18_callback.fp16.ipynb 50
+def grad_overflow(pgs:list)->bool: 
+    "Tests all fp16 parameters in pgs for gradient overflow"
+    for pg in pgs:
+        for p in pg:
+            if p.grad is not None and test_overflow(p.grad.data): return True
+    return False
+
+# %% ../../nbs/18_callback.fp16.ipynb 53
+def copy_clone(d):
+    return {k:(v.detach().clone().float() if isinstance(v,Tensor) else v) for k,v in d.items()}
+
+# %% ../../nbs/18_callback.fp16.ipynb 54
+def _copy_state(opt, pgs1, pgs2):
+    opt.param_lists = pgs2
+    for pg1,pg2 in zip(pgs1, pgs2):
+        for p1,p2 in zip(pg1, pg2): opt.state[p2] = copy_clone(opt.state.pop(p1, {}))
+
+# %% ../../nbs/18_callback.fp16.ipynb 55
+class ModelToHalf(Callback):
+    "Use with NonNativeMixedPrecision callback (but it needs to run at the very beginning)"
+    order=-50
+    def before_fit(self): self.learn.model = convert_network(self.model, dtype=torch.float16)
+    def after_fit (self): self.learn.model = convert_network(self.model, dtype=torch.float32)
+
+# %% ../../nbs/18_callback.fp16.ipynb 56
+@docs
+class NonNativeMixedPrecision(Callback):
+    "Run training in mixed precision"
+    order=10
+    def __init__(self, 
+        loss_scale:int=512, # Non-dynamic loss scale, used to avoid underflow of gradients. 
+        flat_master:bool=False, # Whether to flatten fp32 parameters for performance
+        dynamic:bool=True, # Whether to automatically determine loss scaling
+        max_loss_scale:float=2.**24, # Starting value for dynamic loss scaling
+        div_factor:float=2., # Divide by this on overflow, multiply by this after scale_wait batches
+        scale_wait:int=500, # Number of batches to wait for increasing loss scale
+        clip:float=None, # Value to clip gradients at, max_norm, as in `nn.utils.clip_grad_norm_`
+    ): 
+        assert torch.backends.cudnn.enabled, "Mixed precision training requires cudnn."
+        self.flat_master,self.dynamic,self.max_loss_scale = flat_master,dynamic,max_loss_scale
+        self.div_factor,self.scale_wait,self.clip = div_factor,scale_wait,clip
+        self.loss_scale = max_loss_scale if dynamic else loss_scale
+
+    def before_fit(self):
+        assert self.dls.device.type == 'cuda', "Mixed-precision training requires a GPU, remove the call `to_fp16`"
+        if self.learn.opt is None: self.learn.create_opt()
+        self.model_pgs,self.master_pgs = get_master(self.opt, self.flat_master)
+        self.old_pgs = self.opt.param_lists
+        #Changes the optimizer so that the optimization step is done in FP32.
+        _copy_state(self.learn.opt, self.model_pgs, self.master_pgs)
+        if self.dynamic: self.count = 0
+
+    def before_batch(self): self.learn.xb = to_half(self.xb)
+    def after_pred(self): self.learn.pred = to_float(self.pred)
+    def before_backward(self): self.learn.loss_grad *= self.loss_scale
+
+    def before_step(self):
+        #First, check for an overflow
+        if self.dynamic and grad_overflow(self.model_pgs):
+            self.loss_scale /= self.div_factor
+            self.learn.loss_grad /= self.div_factor #to record correct loss
+            self.model.zero_grad()
+            raise CancelBatchException() #skip step and zero_grad
+        to_master_grads(self.model_pgs, self.master_pgs, self.flat_master)
+        for master_params in self.master_pgs:
+            for param in master_params:
+                if param.grad is not None: param.grad.div_(self.loss_scale)
+        if self.clip is not None:
+            for group in self.master_pgs: nn.utils.clip_grad_norm_(group, self.clip)
+        # Check if it's been long enough without overflow
+        if self.dynamic:
+            self.count += 1
+            if self.count == self.scale_wait:
+                self.count = 0
+                self.loss_scale *= self.div_factor
+
+    def after_step(self):
+        self.model.zero_grad() #Zero the gradients of the model manually (optimizer disconnected)
+        to_model_params(self.model_pgs, self.master_pgs, self.flat_master)
+
+    def after_batch(self):
+        if self.training: self.learn.loss_grad /= self.loss_scale  #Log correct loss
+    def after_fit(self):
+        if not hasattr(self,'master_pgs'): return
+        _copy_state(self.learn.opt, self.master_pgs, self.model_pgs)
+        self.learn.opt.param_lists  = self.old_pgs
+        delattr(self, "master_pgs")
+        delattr(self, "model_pgs")
+        delattr(self, "old_pgs")
+
+    _docs = dict(before_fit="Put the model in FP16 and prepare the two copies of the parameters",
+                 before_batch="Put the input in FP16",
+                 after_pred="Put the output back to FP32 so that the loss is computed in FP32",
+                 before_backward="Apply loss scaling to avoid gradient underflow",
+                 before_step="Update and apply dynamic loss scaling, move gradients to fp32, apply gradient clipping",
+                 after_step="Zero fp16 grads and update fp16 params with fp32 params. ",
+                 after_batch="Ensure loss is logged correctly",
+                 after_fit="Put the model back in FP32")
+
+# %% ../../nbs/18_callback.fp16.ipynb 60
+@patch
+@delegates(NonNativeMixedPrecision.__init__)
+def to_non_native_fp16(self:Learner, **kwargs): return self.add_cbs([ModelToHalf(), NonNativeMixedPrecision(**kwargs)])
+
+# %% ../../nbs/18_callback.fp16.ipynb 63
+@patch
+def to_non_native_fp32(self: Learner): return self.remove_cbs([ModelToHalf, NonNativeMixedPrecision])

fastai/callback/hook.py

@@ -0,0 +1,281 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/15_callback.hook.ipynb.
+
+# %% ../../nbs/15_callback.hook.ipynb 1
+from __future__ import annotations
+from ..basics import *
+
+# %% auto 0
+__all__ = ['Hook', 'hook_output', 'Hooks', 'hook_outputs', 'dummy_eval', 'model_sizes', 'num_features_model', 'has_params',
+           'HookCallback', 'total_params', 'layer_info', 'module_summary', 'ActivationStats']
+
+# %% ../../nbs/15_callback.hook.ipynb 13
+@docs
+class Hook():
+    "Create a hook on `m` with `hook_func`."
+    def __init__(self, m, hook_func, is_forward=True, detach=True, cpu=False, gather=False):
+        store_attr('hook_func,detach,cpu,gather')
+        f = m.register_forward_hook if is_forward else m.register_backward_hook
+        self.hook = f(self.hook_fn)
+        self.stored,self.removed = None,False
+
+    def hook_fn(self, module, input, output):
+        "Applies `hook_func` to `module`, `input`, `output`."
+        if self.detach:
+            input,output = to_detach(input, cpu=self.cpu, gather=self.gather),to_detach(output, cpu=self.cpu, gather=self.gather)
+        self.stored = self.hook_func(module, input, output)
+
+    def remove(self):
+        "Remove the hook from the model."
+        if not self.removed:
+            self.hook.remove()
+            self.removed=True
+
+    def __enter__(self, *args): return self
+    def __exit__(self, *args): self.remove()
+
+    _docs = dict(__enter__="Register the hook",
+                 __exit__="Remove the hook")
+
+# %% ../../nbs/15_callback.hook.ipynb 25
+def _hook_inner(m,i,o): return o if isinstance(o,Tensor) or is_listy(o) else list(o)
+
+def hook_output(module, detach=True, cpu=False, grad=False):
+    "Return a `Hook` that stores activations of `module` in `self.stored`"
+    return Hook(module, _hook_inner, detach=detach, cpu=cpu, is_forward=not grad)
+
+# %% ../../nbs/15_callback.hook.ipynb 30
+@docs
+class Hooks():
+    "Create several hooks on the modules in `ms` with `hook_func`."
+    def __init__(self, ms, hook_func, is_forward=True, detach=True, cpu=False):
+        self.hooks = [Hook(m, hook_func, is_forward, detach, cpu) for m in ms]
+
+    def __getitem__(self,i): return self.hooks[i]
+    def __len__(self):       return len(self.hooks)
+    def __iter__(self):      return iter(self.hooks)
+    @property
+    def stored(self):        return L(o.stored for o in self)
+
+    def remove(self):
+        "Remove the hooks from the model."
+        for h in self.hooks: h.remove()
+
+    def __enter__(self, *args): return self
+    def __exit__ (self, *args): self.remove()
+
+    _docs = dict(stored = "The states saved in each hook.",
+                 __enter__="Register the hooks",
+                 __exit__="Remove the hooks")
+
+# %% ../../nbs/15_callback.hook.ipynb 39
+def hook_outputs(modules, detach=True, cpu=False, grad=False):
+    "Return `Hooks` that store activations of all `modules` in `self.stored`"
+    return Hooks(modules, _hook_inner, detach=detach, cpu=cpu, is_forward=not grad)
+
+# %% ../../nbs/15_callback.hook.ipynb 43
+def dummy_eval(m, size=(64,64)):
+    "Evaluate `m` on a dummy input of a certain `size`"
+    ch_in = in_channels(m)
+    x = one_param(m).new(1, ch_in, *size).requires_grad_(False).uniform_(-1.,1.)
+    with torch.no_grad(): return m.eval()(x)
+
+# %% ../../nbs/15_callback.hook.ipynb 44
+def model_sizes(m, size=(64,64)):
+    "Pass a dummy input through the model `m` to get the various sizes of activations."
+    with hook_outputs(m) as hooks:
+        _ = dummy_eval(m, size=size)
+        return [o.stored.shape for o in hooks]
+
+# %% ../../nbs/15_callback.hook.ipynb 46
+def num_features_model(m):
+    "Return the number of output features for `m`."
+    sz,ch_in = 32,in_channels(m)
+    while True:
+        #Trying for a few sizes in case the model requires a big input size.
+        try:
+            return model_sizes(m, (sz,sz))[-1][1]
+        except Exception as e:
+            sz *= 2
+            if sz > 2048: raise e
+
+# %% ../../nbs/15_callback.hook.ipynb 50
+def has_params(m):
+    "Check if `m` has at least one parameter"
+    return len(list(m.parameters())) > 0
+
+# %% ../../nbs/15_callback.hook.ipynb 52
+@funcs_kwargs
+class HookCallback(Callback):
+    "`Callback` that can be used to register hooks on `modules`"
+    _methods = ["hook"]
+    hook = noops
+    def __init__(self, modules=None, every=None, remove_end=True, is_forward=True, detach=True, cpu=True, include_paramless=False , **kwargs):
+        store_attr('modules,every,remove_end,is_forward,detach,cpu, include_paramless')
+        assert not kwargs
+
+    def before_fit(self):
+        "Register the `Hooks` on `self.modules`."
+        if self.modules is None: self.modules = [m for m in flatten_model(self.model) if self.include_paramless or has_params(m)]
+        if self.every is None: self._register()
+
+    def before_batch(self):
+        if self.every is None: return
+        if self.training and self.train_iter%self.every==0: self._register()
+
+    def after_batch(self):
+        if self.every is None: return
+        if self.training and self.train_iter%self.every==0: self._remove()
+
+    def after_fit(self):
+        "Remove the `Hooks`."
+        if self.remove_end: self._remove()
+
+    def _register(self): self.hooks = Hooks(self.modules, self.hook, self.is_forward, self.detach, self.cpu)
+    def _remove(self):
+        if getattr(self, 'hooks', None): self.hooks.remove()
+
+    def __del__(self): self._remove()
+
+# %% ../../nbs/15_callback.hook.ipynb 59
+def total_params(m):
+    "Give the number of parameters of a module and if it's trainable or not"
+    params = sum([p.numel() for p in m.parameters()])
+    trains = [p.requires_grad for p in m.parameters()]
+    return params, (False if len(trains)==0 else trains[0])
+
+# %% ../../nbs/15_callback.hook.ipynb 61
+def layer_info(learn, *xb):
+    "Return layer infos of `model` on `xb` (only support batch first inputs)"
+    def _track(m, i, o): 
+        params, trainable, shape = '', '', ''
+        same = any((isinstance(x[0], torch.Tensor) and x[0].shape[1:] == x[1].shape for x in zip(i, o)))
+        shape = apply(lambda x: x.shape, o)
+        if hasattr(m, 'weight'): # non activation layer
+            params, trainable = total_params(m)
+        return (type(m).__name__, params, trainable, shape, same)
+            
+    with Hooks(flatten_model(learn.model), _track) as h:
+        batch = apply(lambda o:o[:1], xb)
+        train_only_cbs = [cb for cb in learn.cbs if hasattr(cb, '_only_train_loop')]
+        with learn.removed_cbs(train_only_cbs), learn.no_logging(), learn as l:
+            r = l.get_preds(dl=[batch], inner=True, reorder=False)
+        return h.stored
+
+# %% ../../nbs/15_callback.hook.ipynb 66
+def _get_shapes(o, bs): 
+    inp = o[first(o)] if (isinstance(o, dict)) else o
+    return ' x '.join([str(bs)] + [str(t) for t in inp[1:]])
+
+def _print_shapes(o, bs):
+    if isinstance(o, torch.Size): return _get_shapes(o, bs)
+    elif isinstance(o, tuple): return _get_shapes(o[0], bs)
+    else: return str([_print_shapes(x, bs) for x in o])
+
+# %% ../../nbs/15_callback.hook.ipynb 67
+def module_summary(learn, *xb):
+    "Print a summary of `model` using `xb`"
+    #Individual parameters wrapped in ParameterModule aren't called through the hooks in `layer_info`,
+    #  thus are not counted inside the summary
+    #TODO: find a way to have them counted in param number somehow
+    infos = layer_info(learn, *xb)
+    n,bs = 76,find_bs(xb)
+    inp_sz = _print_shapes(apply(lambda x:x.shape, xb), bs)
+    res = f"{type(learn.model).__name__} (Input shape: {inp_sz})\n"
+    res += "=" * n + "\n"
+    res += f"{'Layer (type)':<20} {'Output Shape':<20} {'Param #':<10} {'Trainable':<10}\n"
+    res += "=" * n
+    ps,trn_ps,j = 0,0,0
+    infos = [o for o in infos if o is not None] #see comment in previous cell
+    prev_sz = None
+    for typ,np,trn,sz,chnged in infos:
+        if sz is None: continue
+        if j == 0:
+            res += f'\n{"":<20} {_print_shapes(sz, bs)[:19]:<20}' # to avoid a double line at the top
+        if not chnged and not prev_sz == sz and j > 0: res += "\n" + "_" * n + "\n" + f'{"":<20} {_print_shapes(sz, bs)[:19]:<20}'
+        j = 1
+        res += f"\n{typ:<20} {'':<20} {np:<10} {str(trn):<10}"
+        if np != '':
+            ps += np
+            if trn: trn_ps += np
+        prev_sz = sz
+    res += "\n" + "_" * n + "\n"
+    res += f"\nTotal params: {ps:,}\n"
+    res += f"Total trainable params: {trn_ps:,}\n"
+    res += f"Total non-trainable params: {ps - trn_ps:,}\n\n"
+    return PrettyString(res)
+
+# %% ../../nbs/15_callback.hook.ipynb 68
+@patch
+def summary(self:Learner):
+    "Print a summary of the model, optimizer and loss function."
+    xb = self.dls.train.one_batch()[:getattr(self.dls.train, "n_inp", 1)]
+    res = module_summary(self, *xb)
+    res += f"Optimizer used: {self.opt_func}\nLoss function: {self.loss_func}\n\n"
+    if self.opt is not None:
+        res += f"Model " + ("unfrozen\n\n" if self.opt.frozen_idx==0 else f"frozen up to parameter group #{self.opt.frozen_idx}\n\n")
+    res += "Callbacks:\n" + '\n'.join(f"  - {cb}" for cb in self.cbs.sorted('order'))
+    return PrettyString(res)
+
+# %% ../../nbs/15_callback.hook.ipynb 74
+@delegates()
+class ActivationStats(HookCallback):
+    "Callback that record the mean and std of activations."
+    order=-20
+    def __init__(self, with_hist=False, **kwargs):
+        super().__init__(**kwargs)
+        self.with_hist = with_hist
+
+    def before_fit(self):
+        "Initialize stats."
+        super().before_fit()
+        self.stats = L()
+
+    def hook(self, m, i, o):
+        if isinstance(o, tuple): return self.hook_multi_ouput(o)
+        o = o.float()
+        res = {'mean': o.mean().item(), 'std': o.std().item(),
+               'near_zero': (o<=0.05).long().sum().item()/o.numel()}
+        if self.with_hist: res['hist'] = o.histc(40,0,10)
+        return res
+    
+    def hook_multi_ouput(self,o_tuple):
+        "For outputs of RNN which are [nested] tuples of tensors"
+        res = []
+        for o in self._flatten_tuple(o_tuple):
+            if not(isinstance(o, Tensor)): continue
+            res.append(self.hook(None, None, o))
+        return res
+
+    def _flatten_tuple(self, o_tuple):
+        "Recursively flatten a [nested] tuple"
+        res = []
+        for it in o_tuple:
+            if isinstance(it, tuple): res += self._flatten_tuple(it)
+            else: res += [it]
+        return tuple(res)
+
+    def after_batch(self):
+        "Take the stored results and puts it in `self.stats`"
+        if self.training and (self.every is None or self.train_iter%self.every == 0): self.stats.append(self.hooks.stored)
+        super().after_batch()
+
+    def layer_stats(self, idx):
+        lstats = self.stats.itemgot(idx)
+        return L(lstats.itemgot(o) for o in ('mean','std','near_zero'))
+
+    def hist(self, idx):
+        res = self.stats.itemgot(idx).itemgot('hist')
+        return torch.stack(tuple(res)).t().float().log1p()
+
+    def color_dim(self, idx, figsize=(10,5), ax=None):
+        "The 'colorful dimension' plot"
+        res = self.hist(idx)
+        if ax is None: ax = subplots(figsize=figsize)[1][0]
+        ax.imshow(res, origin='lower')
+        ax.axis('off')
+
+    def plot_layer_stats(self, idx):
+        _,axs = subplots(1, 3, figsize=(12,3))
+        for o,ax,title in zip(self.layer_stats(idx),axs,('mean','std','% near zero')):
+            ax.plot(o)
+            ax.set_title(title)

fastai/callback/mixup.py

@@ -0,0 +1,111 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/19_callback.mixup.ipynb.
+
+# %% ../../nbs/19_callback.mixup.ipynb 2
+from __future__ import annotations
+from ..basics import *
+from torch.distributions.beta import Beta
+
+# %% auto 0
+__all__ = ['reduce_loss', 'MixHandler', 'MixUp', 'CutMix']
+
+# %% ../../nbs/19_callback.mixup.ipynb 6
+def reduce_loss(
+    loss:Tensor, 
+    reduction:str='mean' # PyTorch loss reduction
+)->Tensor:
+    "Reduce the loss based on `reduction`"
+    return loss.mean() if reduction == 'mean' else loss.sum() if reduction == 'sum' else loss
+
+# %% ../../nbs/19_callback.mixup.ipynb 7
+class MixHandler(Callback):
+    "A handler class for implementing `MixUp` style scheduling"
+    run_valid = False
+    def __init__(self, 
+        alpha:float=0.5 # Determine `Beta` distribution in range (0.,inf]
+    ):
+        self.distrib = Beta(tensor(alpha), tensor(alpha))
+
+    def before_train(self):
+        "Determine whether to stack y"
+        self.stack_y = getattr(self.learn.loss_func, 'y_int', False)
+        if self.stack_y: self.old_lf,self.learn.loss_func = self.learn.loss_func,self.lf
+
+    def after_train(self):
+        "Set the loss function back to the previous loss"
+        if self.stack_y: self.learn.loss_func = self.old_lf
+
+    def after_cancel_train(self):
+        "If training is canceled, still set the loss function back"
+        self.after_train()
+
+    def after_cancel_fit(self):
+        "If fit is canceled, still set the loss function back"
+        self.after_train()
+
+    def lf(self, pred, *yb):
+        "lf is a loss function that applies the original loss function on both outputs based on `self.lam`"
+        if not self.training: return self.old_lf(pred, *yb)
+        with NoneReduce(self.old_lf) as lf:
+            loss = torch.lerp(lf(pred,*self.yb1), lf(pred,*yb), self.lam)
+        return reduce_loss(loss, getattr(self.old_lf, 'reduction', 'mean'))
+
+# %% ../../nbs/19_callback.mixup.ipynb 10
+class MixUp(MixHandler):
+    "Implementation of https://arxiv.org/abs/1710.09412"
+    def __init__(self, 
+        alpha:float=.4 # Determine `Beta` distribution in range (0.,inf]
+    ): 
+        super().__init__(alpha)
+        
+    def before_batch(self):
+        "Blend xb and yb with another random item in a second batch (xb1,yb1) with `lam` weights"
+        lam = self.distrib.sample((self.y.size(0),)).squeeze().to(self.x.device)
+        lam = torch.stack([lam, 1-lam], 1)
+        self.lam = lam.max(1)[0]
+        shuffle = torch.randperm(self.y.size(0)).to(self.x.device)
+        xb1,self.yb1 = tuple(L(self.xb).itemgot(shuffle)),tuple(L(self.yb).itemgot(shuffle))
+        nx_dims = len(self.x.size())
+        self.learn.xb = tuple(L(xb1,self.xb).map_zip(torch.lerp,weight=unsqueeze(self.lam, n=nx_dims-1)))
+
+        if not self.stack_y:
+            ny_dims = len(self.y.size())
+            self.learn.yb = tuple(L(self.yb1,self.yb).map_zip(torch.lerp,weight=unsqueeze(self.lam, n=ny_dims-1)))
+
+# %% ../../nbs/19_callback.mixup.ipynb 21
+class CutMix(MixHandler): 
+    "Implementation of https://arxiv.org/abs/1905.04899"
+    def __init__(self, 
+        alpha:float=1. # Determine `Beta` distribution in range (0.,inf]
+    ): 
+        super().__init__(alpha)
+        
+    def before_batch(self):
+        "Add `rand_bbox` patches with size based on `lam` and location chosen randomly."
+        bs, _, H, W = self.x.size()
+        self.lam = self.distrib.sample((1,)).to(self.x.device)
+        shuffle = torch.randperm(bs).to(self.x.device)
+        xb1,self.yb1 = self.x[shuffle], tuple((self.y[shuffle],))
+        x1, y1, x2, y2 = self.rand_bbox(W, H, self.lam)
+        self.learn.xb[0][..., y1:y2, x1:x2] = xb1[..., y1:y2, x1:x2]
+        self.lam = (1 - ((x2-x1)*(y2-y1))/float(W*H))
+        if not self.stack_y:
+            ny_dims = len(self.y.size())
+            self.learn.yb = tuple(L(self.yb1,self.yb).map_zip(torch.lerp,weight=unsqueeze(self.lam, n=ny_dims-1)))
+
+    def rand_bbox(self, 
+        W:int, # Width bbox will be
+        H:int, # Height bbox will be
+        lam:Tensor # lambda sample from Beta distribution i.e tensor([0.3647])
+    )->tuple: # Represents the top-left pixel location and the bottom-right pixel location
+        "Give a bounding box location based on the size of the im and a weight"
+        cut_rat = torch.sqrt(1. - lam).to(self.x.device)
+        cut_w = torch.round(W * cut_rat).type(torch.long).to(self.x.device)
+        cut_h = torch.round(H * cut_rat).type(torch.long).to(self.x.device)
+        # uniform
+        cx = torch.randint(0, W, (1,)).to(self.x.device)
+        cy = torch.randint(0, H, (1,)).to(self.x.device)
+        x1 = torch.clamp(cx - cut_w // 2, 0, W)
+        y1 = torch.clamp(cy - cut_h // 2, 0, H)
+        x2 = torch.clamp(cx + cut_w // 2, 0, W)
+        y2 = torch.clamp(cy + cut_h // 2, 0, H)
+        return x1, y1, x2, y2

fastai/callback/neptune.py

@@ -0,0 +1,80 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/70b_callback.neptune.ipynb.
+
+# %% ../../nbs/70b_callback.neptune.ipynb 2
+from __future__ import annotations
+import tempfile
+from ..basics import *
+from ..learner import Callback
+
+# %% auto 0
+__all__ = ['NeptuneCallback']
+
+# %% ../../nbs/70b_callback.neptune.ipynb 12
+import neptune
+
+# %% ../../nbs/70b_callback.neptune.ipynb 13
+class NeptuneCallback(Callback):
+    "Log losses, metrics, model weights, model architecture summary to neptune"
+    order = Recorder.order+1
+    def __init__(self, log_model_weights=True, keep_experiment_running=False):
+        self.log_model_weights = log_model_weights
+        self.keep_experiment_running = keep_experiment_running
+        self.experiment = None
+
+        if neptune.project is None:
+            raise ValueError('You did not initialize project in neptune.\n',
+                             'Please invoke `neptune.init("USERNAME/PROJECT_NAME")` before this callback.')
+
+    def before_fit(self):
+        try:
+            self.experiment = neptune.get_experiment()
+        except ValueError:
+            print('No active experiment. Please invoke `neptune.create_experiment()` before this callback.')
+
+        try:
+            self.experiment.set_property('n_epoch', str(self.learn.n_epoch))
+            self.experiment.set_property('model_class', str(type(self.learn.model)))
+        except: print(f'Did not log all properties. Check properties in the {neptune.get_experiment()}.')
+
+        try:
+            with tempfile.NamedTemporaryFile(mode='w') as f:
+                with open(f.name, 'w') as g: g.write(repr(self.learn.model))
+                self.experiment.log_artifact(f.name, 'model_summary.txt')
+        except: print('Did not log model summary. Check if your model is PyTorch model.')
+
+        if self.log_model_weights and not hasattr(self.learn, 'save_model'):
+            print('Unable to log model to Neptune.\n',
+                  'Use "SaveModelCallback" to save model checkpoints that will be logged to Neptune.')
+
+    def after_batch(self):
+        # log loss and opt.hypers
+        if self.learn.training:
+            self.experiment.log_metric('batch__smooth_loss', self.learn.smooth_loss)
+            self.experiment.log_metric('batch__loss', self.learn.loss)
+            self.experiment.log_metric('batch__train_iter', self.learn.train_iter)
+            for i, h in enumerate(self.learn.opt.hypers):
+                for k, v in h.items(): self.experiment.log_metric(f'batch__opt.hypers.{k}', v)
+
+    def after_epoch(self):
+        # log metrics
+        for n, v in zip(self.learn.recorder.metric_names, self.learn.recorder.log):
+            if n not in ['epoch', 'time']: self.experiment.log_metric(f'epoch__{n}', v)
+            if n == 'time': self.experiment.log_text(f'epoch__{n}', str(v))
+
+        # log model weights
+        if self.log_model_weights and hasattr(self.learn, 'save_model'):
+            if self.learn.save_model.every_epoch:
+                _file = join_path_file(f'{self.learn.save_model.fname}_{self.learn.save_model.epoch}',
+                                       self.learn.path / self.learn.model_dir, ext='.pth')
+            else:
+                _file = join_path_file(self.learn.save_model.fname,
+                                       self.learn.path / self.learn.model_dir, ext='.pth')
+            self.experiment.log_artifact(_file)
+
+    def after_fit(self):
+        if not self.keep_experiment_running:
+            try: self.experiment.stop()
+            except: print('No neptune experiment to stop.')
+        else:
+            print(f'Your experiment (id: {self.experiment.id}, name: {self.experiment.name}) is left in the running state.\n',
+                  'You can log more data to it, like this: `neptune.log_metric()`')

fastai/callback/preds.py

@@ -0,0 +1,18 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/18b_callback.preds.ipynb.
+
+# %% ../../nbs/18b_callback.preds.ipynb 2
+from __future__ import annotations
+from ..basics import *
+
+# %% auto 0
+__all__ = ['MCDropoutCallback']
+
+# %% ../../nbs/18b_callback.preds.ipynb 6
+class MCDropoutCallback(Callback):
+    def before_validate(self):
+        for m in [m for m in flatten_model(self.model) if 'dropout' in m.__class__.__name__.lower()]:
+            m.train()
+    
+    def after_validate(self):
+        for m in [m for m in flatten_model(self.model) if 'dropout' in m.__class__.__name__.lower()]:
+            m.eval()

fastai/callback/progress.py

@@ -0,0 +1,124 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/16_callback.progress.ipynb.
+
+# %% ../../nbs/16_callback.progress.ipynb 1
+from __future__ import annotations
+from ..basics import *
+
+# %% auto 0
+__all__ = ['ProgressCallback', 'ShowGraphCallback', 'CSVLogger']
+
+# %% ../../nbs/16_callback.progress.ipynb 7
+@docs
+class ProgressCallback(Callback):
+    "A `Callback` to handle the display of progress bars"
+    order,_stateattrs = 60,('mbar','pbar')
+
+    def before_fit(self):
+        assert hasattr(self.learn, 'recorder')
+        if self.create_mbar: self.mbar = master_bar(list(range(self.n_epoch)))
+        if self.learn.logger != noop:
+            self.old_logger,self.learn.logger = self.logger,self._write_stats
+            self._write_stats(self.recorder.metric_names)
+        else: self.old_logger = noop
+
+    def before_epoch(self):
+        if getattr(self, 'mbar', False): self.mbar.update(self.epoch)
+
+    def before_train(self):    self._launch_pbar()
+    def before_validate(self): self._launch_pbar()
+    def after_train(self):     self.pbar.on_iter_end()
+    def after_validate(self):  self.pbar.on_iter_end()
+    def after_batch(self):
+        self.pbar.update(self.iter+1)
+        if hasattr(self, 'smooth_loss'): self.pbar.comment = f'{self.smooth_loss.item():.4f}'
+
+    def _launch_pbar(self):
+        self.pbar = progress_bar(self.dl, parent=getattr(self, 'mbar', None), leave=False)
+        self.pbar.update(0)
+
+    def after_fit(self):
+        if getattr(self, 'mbar', False):
+            self.mbar.on_iter_end()
+            delattr(self, 'mbar')
+        if hasattr(self, 'old_logger'): self.learn.logger = self.old_logger
+
+    def _write_stats(self, log):
+        if getattr(self, 'mbar', False): self.mbar.write([f'{l:.6f}' if isinstance(l, float) else str(l) for l in log], table=True)
+
+    _docs = dict(before_fit="Setup the master bar over the epochs",
+                 before_epoch="Update the master bar",
+                 before_train="Launch a progress bar over the training dataloader",
+                 before_validate="Launch a progress bar over the validation dataloader",
+                 after_train="Close the progress bar over the training dataloader",
+                 after_validate="Close the progress bar over the validation dataloader",
+                 after_batch="Update the current progress bar",
+                 after_fit="Close the master bar")
+
+if not hasattr(defaults, 'callbacks'): defaults.callbacks = [TrainEvalCallback, Recorder, ProgressCallback]
+elif ProgressCallback not in defaults.callbacks: defaults.callbacks.append(ProgressCallback)
+
+# %% ../../nbs/16_callback.progress.ipynb 9
+@patch
+@contextmanager
+def no_bar(self:Learner):
+    "Context manager that deactivates the use of progress bars"
+    has_progress = hasattr(self, 'progress')
+    if has_progress: self.remove_cb(self.progress)
+    try: yield self
+    finally:
+        if has_progress: self.add_cb(ProgressCallback())
+
+# %% ../../nbs/16_callback.progress.ipynb 22
+class ShowGraphCallback(Callback):
+    "Update a graph of training and validation loss"
+    order,run_valid=65,False
+
+    def before_fit(self):
+        self.run = not hasattr(self.learn, 'lr_finder') and not hasattr(self, "gather_preds")
+        if not(self.run): return
+        self.nb_batches = []
+        assert hasattr(self.learn, 'progress')
+
+    def after_train(self): self.nb_batches.append(self.train_iter)
+
+    def after_epoch(self):
+        "Plot validation loss in the pbar graph"
+        if not self.nb_batches: return
+        rec = self.learn.recorder
+        iters = range_of(rec.losses)
+        val_losses = [v[1] for v in rec.values]
+        x_bounds = (0, (self.n_epoch - len(self.nb_batches)) * self.nb_batches[0] + len(rec.losses))
+        y_bounds = (0, max((max(Tensor(rec.losses)), max(Tensor(val_losses)))))
+        self.progress.mbar.update_graph([(iters, rec.losses), (self.nb_batches, val_losses)], x_bounds, y_bounds)
+
+# %% ../../nbs/16_callback.progress.ipynb 26
+class CSVLogger(Callback):
+    "Log the results displayed in `learn.path/fname`"
+    order=60
+    def __init__(self, fname='history.csv', append=False):
+        self.fname,self.append = Path(fname),append
+
+    def read_log(self):
+        "Convenience method to quickly access the log."
+        return pd.read_csv(self.path/self.fname)
+
+    def before_fit(self):
+        "Prepare file with metric names."
+        if hasattr(self, "gather_preds"): return
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.file = (self.path/self.fname).open('a' if self.append else 'w')
+        self.file.write(','.join(self.recorder.metric_names) + '\n')
+        self.old_logger,self.learn.logger = self.logger,self._write_line
+
+    def _write_line(self, log):
+        "Write a line with `log` and call the old logger."
+        self.file.write(','.join([str(t) for t in log]) + '\n')
+        self.file.flush()
+        os.fsync(self.file.fileno())
+        self.old_logger(log)
+
+    def after_fit(self):
+        "Close the file and clean up."
+        if hasattr(self, "gather_preds"): return
+        self.file.close()
+        self.learn.logger = self.old_logger

fastai/callback/rnn.py

@@ -0,0 +1,42 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/34_callback.rnn.ipynb.
+
+# %% ../../nbs/34_callback.rnn.ipynb 1
+from __future__ import annotations
+from ..basics import *
+
+# %% auto 0
+__all__ = ['ModelResetter', 'RNNCallback', 'RNNRegularizer', 'rnn_cbs']
+
+# %% ../../nbs/34_callback.rnn.ipynb 5
+@docs
+class ModelResetter(Callback):
+    "`Callback` that resets the model at each validation/training step"
+    def before_train(self):    self.model.reset()
+    def before_validate(self): self.model.reset()
+    def after_fit(self):       self.model.reset()
+    _docs = dict(before_train="Reset the model before training",
+                 before_validate="Reset the model before validation",
+                 after_fit="Reset the model after fitting")
+
+# %% ../../nbs/34_callback.rnn.ipynb 6
+class RNNCallback(Callback):
+    "Save the raw and dropped-out outputs and only keep the true output for loss computation"
+    def after_pred(self): self.learn.pred,self.raw_out,self.out = [o[-1] if is_listy(o) else o for o in self.pred]
+
+# %% ../../nbs/34_callback.rnn.ipynb 7
+class RNNRegularizer(Callback):
+    "Add AR and TAR regularization"
+    order,run_valid = RNNCallback.order+1,False
+    def __init__(self, alpha=0., beta=0.): store_attr()
+    def after_loss(self):
+        if not self.training: return
+        if self.alpha: self.learn.loss_grad += self.alpha * self.rnn.out.float().pow(2).mean()
+        if self.beta:
+            h = self.rnn.raw_out
+            if len(h)>1: self.learn.loss_grad += self.beta * (h[:,1:] - h[:,:-1]).float().pow(2).mean()
+
+# %% ../../nbs/34_callback.rnn.ipynb 8
+def rnn_cbs(alpha=0., beta=0.):
+    "All callbacks needed for (optionally regularized) RNN training"
+    reg = [RNNRegularizer(alpha=alpha, beta=beta)] if alpha or beta else []
+    return [ModelResetter(), RNNCallback()] + reg

fastai/callback/schedule.py

@@ -0,0 +1,314 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/14_callback.schedule.ipynb.
+
+# %% ../../nbs/14_callback.schedule.ipynb 2
+from __future__ import annotations
+from ..basics import *
+from .tracker import SaveModelCallback
+
+# %% auto 0
+__all__ = ['annealer', 'sched_lin', 'sched_cos', 'sched_no', 'sched_exp', 'SchedLin', 'SchedCos', 'SchedNo', 'SchedExp',
+           'SchedPoly', 'combine_scheds', 'combined_cos', 'ParamScheduler', 'LRFinder', 'valley', 'slide', 'minimum',
+           'steep', 'SuggestionMethod']
+
+# %% ../../nbs/14_callback.schedule.ipynb 3
+_all_ = ['SuggestionMethod']
+
+# %% ../../nbs/14_callback.schedule.ipynb 8
+class _Annealer:
+    def __init__(self, f, start, end): store_attr('f,start,end')
+    def __call__(self, pos): return self.f(self.start, self.end, pos)
+
+# %% ../../nbs/14_callback.schedule.ipynb 9
+def annealer(f):
+    "Decorator to make `f` return itself partially applied."
+    @functools.wraps(f)
+    def _inner(start, end): return _Annealer(f, start, end)
+    return _inner
+
+# %% ../../nbs/14_callback.schedule.ipynb 11
+#TODO Jeremy, make this pickle
+#@annealer
+#def SchedLin(start, end, pos): return start + pos*(end-start)
+#@annealer
+#def SchedCos(start, end, pos): return start + (1 + math.cos(math.pi*(1-pos))) * (end-start) / 2
+#@annealer
+#def SchedNo (start, end, pos): return start
+#@annealer
+#def SchedExp(start, end, pos): return start * (end/start) ** pos
+#
+#SchedLin.__doc__ = "Linear schedule function from `start` to `end`"
+#SchedCos.__doc__ = "Cosine schedule function from `start` to `end`"
+#SchedNo .__doc__ = "Constant schedule function with `start` value"
+#SchedExp.__doc__ = "Exponential schedule function from `start` to `end`"
+
+# %% ../../nbs/14_callback.schedule.ipynb 12
+def sched_lin(start, end, pos): return start + pos*(end-start)
+def sched_cos(start, end, pos): return start + (1 + math.cos(math.pi*(1-pos))) * (end-start) / 2
+def sched_no (start, end, pos): return start
+def sched_exp(start, end, pos): return start * (end/start) ** pos
+
+def SchedLin(start, end): return _Annealer(sched_lin, start, end)
+def SchedCos(start, end): return _Annealer(sched_cos, start, end)
+def SchedNo (start, end): return _Annealer(sched_no,  start, end)
+def SchedExp(start, end): return _Annealer(sched_exp, start, end)
+
+SchedLin.__doc__ = "Linear schedule function from `start` to `end`"
+SchedCos.__doc__ = "Cosine schedule function from `start` to `end`"
+SchedNo .__doc__ = "Constant schedule function with `start` value"
+SchedExp.__doc__ = "Exponential schedule function from `start` to `end`"
+
+# %% ../../nbs/14_callback.schedule.ipynb 15
+def SchedPoly(start, end, power):
+    "Polynomial schedule (of `power`) function from `start` to `end`"
+    def _inner(pos): return start + (end - start) * pos ** power
+    return _inner
+
+# %% ../../nbs/14_callback.schedule.ipynb 28
+def combine_scheds(pcts, scheds):
+    "Combine `scheds` according to `pcts` in one function"
+    assert sum(pcts) == 1.
+    pcts = tensor([0] + L(pcts))
+    assert torch.all(pcts >= 0)
+    pcts = torch.cumsum(pcts, 0)
+    pct_lim = len(pcts) - 2
+    def _inner(pos):
+        idx = min((pos >= pcts).nonzero().max(), pct_lim)
+        actual_pos = (pos-pcts[idx]) / (pcts[idx+1]-pcts[idx])
+        return scheds[idx](actual_pos.item())
+    return _inner
+
+# %% ../../nbs/14_callback.schedule.ipynb 33
+def combined_cos(pct, start, middle, end):
+    "Return a scheduler with cosine annealing from `start`→`middle` & `middle`→`end`"
+    return combine_scheds([pct,1-pct], [SchedCos(start, middle), SchedCos(middle, end)])
+
+# %% ../../nbs/14_callback.schedule.ipynb 38
+@docs
+class ParamScheduler(Callback):
+    "Schedule hyper-parameters according to `scheds`"
+    order,run_valid = 60,False
+
+    def __init__(self, scheds): self.scheds = scheds
+    def before_fit(self): self.hps = {p:[] for p in self.scheds.keys()}
+    def before_batch(self): self._update_val(self.pct_train)
+
+    def _update_val(self, pct):
+        for n,f in self.scheds.items(): self.opt.set_hyper(n, f(pct))
+
+    def after_batch(self):
+        for p in self.scheds.keys(): self.hps[p].append(self.opt.hypers[-1][p])
+
+    def after_fit(self):
+        if hasattr(self.learn, 'recorder') and hasattr(self, 'hps'): self.recorder.hps = self.hps
+
+    _docs = {"before_fit": "Initialize container for hyper-parameters",
+             "before_batch": "Set the proper hyper-parameters in the optimizer",
+             "after_batch": "Record hyper-parameters of this batch",
+             "after_fit": "Save the hyper-parameters in the recorder if there is one"}
+
+# %% ../../nbs/14_callback.schedule.ipynb 46
+@patch
+def fit_one_cycle(self:Learner, n_epoch, lr_max=None, div=25., div_final=1e5, pct_start=0.25, wd=None,
+                  moms=None, cbs=None, reset_opt=False, start_epoch=0):
+    "Fit `self.model` for `n_epoch` using the 1cycle policy."
+    if self.opt is None: self.create_opt()
+    self.opt.set_hyper('lr', self.lr if lr_max is None else lr_max)
+    lr_max = np.array([h['lr'] for h in self.opt.hypers])
+    scheds = {'lr': combined_cos(pct_start, lr_max/div, lr_max, lr_max/div_final),
+              'mom': combined_cos(pct_start, *(self.moms if moms is None else moms))}
+    self.fit(n_epoch, cbs=ParamScheduler(scheds)+L(cbs), reset_opt=reset_opt, wd=wd, start_epoch=start_epoch)
+
+# %% ../../nbs/14_callback.schedule.ipynb 50
+@patch
+def plot_sched(self:Recorder, keys=None, figsize=None):
+    keys = self.hps.keys() if keys is None else L(keys)
+    rows,cols = (len(keys)+1)//2, min(2, len(keys))
+    figsize = figsize or (6*cols,4*rows)
+    _, axs = plt.subplots(rows, cols, figsize=figsize)
+    axs = axs.flatten() if len(keys) > 1 else L(axs)
+    for p,ax in zip(keys, axs):
+        ax.plot(self.hps[p])
+        ax.set_ylabel(p)
+
+# %% ../../nbs/14_callback.schedule.ipynb 54
+@patch
+def fit_flat_cos(self:Learner, n_epoch, lr=None, div_final=1e5, pct_start=0.75, wd=None,
+                 cbs=None, reset_opt=False, start_epoch=0):
+    "Fit `self.model` for `n_epoch` at flat `lr` before a cosine annealing."
+    if self.opt is None: self.create_opt()
+    self.opt.set_hyper('lr', self.lr if lr is None else lr)
+    lr = np.array([h['lr'] for h in self.opt.hypers])
+    scheds = {'lr': combined_cos(pct_start, lr, lr, lr/div_final)}
+    self.fit(n_epoch, cbs=ParamScheduler(scheds)+L(cbs), reset_opt=reset_opt, wd=wd, start_epoch=0)
+
+# %% ../../nbs/14_callback.schedule.ipynb 57
+@patch
+def fit_sgdr(self:Learner, n_cycles, cycle_len, lr_max=None, cycle_mult=2, cbs=None, reset_opt=False, wd=None,
+             start_epoch=0):
+    "Fit `self.model` for `n_cycles` of `cycle_len` using SGDR."
+    if self.opt is None: self.create_opt()
+    self.opt.set_hyper('lr', self.lr if lr_max is None else lr_max)
+    lr_max = np.array([h['lr'] for h in self.opt.hypers])
+    n_epoch = cycle_len * (cycle_mult**n_cycles-1)//(cycle_mult-1)
+    pcts = [cycle_len * cycle_mult**i / n_epoch for i in range(n_cycles)]
+    scheds = [SchedCos(lr_max, 0) for _ in range(n_cycles)]
+    scheds = {'lr': combine_scheds(pcts, scheds)}
+    self.fit(n_epoch, cbs=ParamScheduler(scheds)+L(cbs), reset_opt=reset_opt, wd=wd, start_epoch=start_epoch)
+
+# %% ../../nbs/14_callback.schedule.ipynb 60
+@patch
+@delegates(Learner.fit_one_cycle)
+def fine_tune(self:Learner, epochs, base_lr=2e-3, freeze_epochs=1, lr_mult=100,
+              pct_start=0.3, div=5.0, **kwargs):
+    "Fine tune with `Learner.freeze` for `freeze_epochs`, then with `Learner.unfreeze` for `epochs`, using discriminative LR."
+    self.freeze()
+    self.fit_one_cycle(freeze_epochs, slice(base_lr), pct_start=0.99, **kwargs)
+    base_lr /= 2
+    self.unfreeze()
+    self.fit_one_cycle(epochs, slice(base_lr/lr_mult, base_lr), pct_start=pct_start, div=div, **kwargs)
+
+# %% ../../nbs/14_callback.schedule.ipynb 67
+@docs
+class LRFinder(ParamScheduler):
+    "Training with exponentially growing learning rate"
+    def __init__(self, start_lr=1e-7, end_lr=10, num_it=100, stop_div=True):
+        if num_it < 6: num_it = 6
+        self.scheds = {'lr': [SchedExp(s, e) for (s,e) in zip(start_lr,end_lr)
+                             ] if is_listy(start_lr) else SchedExp(start_lr, end_lr)}
+        self.num_it,self.stop_div = num_it,stop_div
+
+    def before_fit(self):
+        super().before_fit()
+        path = self.path/self.model_dir
+        path.mkdir(parents=True, exist_ok=True)
+        self.tmp_d = tempfile.TemporaryDirectory(dir=path)
+        self.tmp_p = Path(self.tmp_d.name).stem
+        self.learn.save(f'{self.tmp_p}/_tmp')
+        self.best_loss = float('inf')
+
+    def before_batch(self): self._update_val(self.train_iter/self.num_it)
+
+    def after_batch(self):
+        super().after_batch()
+        if self.smooth_loss < self.best_loss: self.best_loss = self.smooth_loss
+        if self.smooth_loss > 4*self.best_loss and self.stop_div: raise CancelFitException()
+        if self.train_iter >= self.num_it: raise CancelFitException()
+
+    def before_validate(self): raise CancelValidException()
+
+    def after_fit(self):
+        self.learn.opt.zero_grad() # Needed before detaching the optimizer for future fits
+        tmp_f = self.path/self.model_dir/self.tmp_p/'_tmp.pth'
+        if tmp_f.exists():
+            self.learn.load(f'{self.tmp_p}/_tmp', with_opt=True)
+            self.tmp_d.cleanup()
+
+    _docs = {"before_fit": "Initialize container for hyper-parameters and save the model",
+             "before_batch": "Set the proper hyper-parameters in the optimizer",
+             "after_batch": "Record hyper-parameters of this batch and potentially stop training",
+             "after_fit": "Save the hyper-parameters in the recorder if there is one and load the original model",
+             "before_validate": "Skip the validation part of training"}
+
+# %% ../../nbs/14_callback.schedule.ipynb 78
+def valley(lrs:list, losses:list, num_it:int):
+    "Suggests a learning rate from the longest valley and returns its index"
+    n = len(losses)
+    max_start, max_end = 0,0
+
+    # find the longest valley
+    lds = [1]*n
+    for i in range(1,n):
+        for j in range(0,i):
+            if (losses[i] < losses[j]) and (lds[i] < lds[j] + 1):
+                lds[i] = lds[j] + 1
+            if lds[max_end] < lds[i]:
+                max_end = i
+                max_start = max_end - lds[max_end]
+    
+    sections = (max_end - max_start) / 3
+    idx = max_start + int(sections) + int(sections/2)
+
+    return float(lrs[idx]), (float(lrs[idx]), losses[idx])
+
+# %% ../../nbs/14_callback.schedule.ipynb 81
+def slide(lrs:list, losses:list, num_it:int, lr_diff:int=15, thresh:float=.005, adjust_value:float=1.):
+    "Suggests a learning rate following an interval slide rule and returns its index"
+    losses = to_np(losses)
+    loss_grad = np.gradient(losses)
+
+    r_idx = -1
+    l_idx = r_idx - lr_diff
+    local_min_lr = lrs[l_idx]
+    while (l_idx >= -len(losses)) and (abs(loss_grad[r_idx] - loss_grad[l_idx]) > thresh):
+        local_min_lr = lrs[l_idx]
+        r_idx -= 1
+        l_idx -= 1
+    
+    suggestion = float(local_min_lr) * adjust_value
+    idx = np.interp(np.log10(suggestion), np.log10(lrs), losses)
+    return suggestion, (suggestion, idx)
+
+# %% ../../nbs/14_callback.schedule.ipynb 84
+def minimum(lrs:list, losses:list, num_it:int):
+    "Suggests a learning rate one-tenth the minumum before divergance and returns its index"
+    lr_min = lrs[losses.argmin()].item()
+    loss_idx = losses[min(range(len(lrs)), key=lambda i: abs(lrs[i]-lr_min))]
+    return lr_min/10, (lr_min, loss_idx)
+
+# %% ../../nbs/14_callback.schedule.ipynb 86
+def steep(lrs:list, losses:list, num_it:int) -> (float, tuple):
+    "Suggests a learning rate when the slope is the steepest and returns its index"
+    grads = (losses[1:]-losses[:-1]) / (lrs[1:].log()-lrs[:-1].log())
+    lr_steep = lrs[grads.argmin()].item()
+    loss_idx = losses[min(range(len(lrs)), key=lambda i: abs(lrs[i]-lr_steep))]
+    return lr_steep, (lr_steep, loss_idx)
+
+# %% ../../nbs/14_callback.schedule.ipynb 88
+@patch
+def plot_lr_find(self:Recorder, skip_end=5, return_fig=True, suggestions=None, nms=None, **kwargs):
+    "Plot the result of an LR Finder test (won't work if you didn't do `learn.lr_find()` before)"
+    lrs    = self.lrs    if skip_end==0 else self.lrs   [:-skip_end]
+    losses = self.losses if skip_end==0 else self.losses[:-skip_end]
+    fig, ax = plt.subplots(1,1)
+    ax.plot(lrs, losses)
+    ax.set_ylabel("Loss")
+    ax.set_xlabel("Learning Rate")
+    ax.set_xscale('log')
+    if suggestions:
+        colors = plt.rcParams['axes.prop_cycle'].by_key()['color'][1:]
+        for (val, idx), nm, color in zip(suggestions, nms, colors):
+            ax.plot(val, idx, 'o', label=nm, c=color)
+        ax.legend(loc='best')
+
+# %% ../../nbs/14_callback.schedule.ipynb 89
+mk_class("SuggestionMethod", **{o.__name__.capitalize():o for o in [valley,slide,minimum,steep]},
+         doc="All possible suggestion methods as convience attributes to get tab-completion and typo-proofing")
+
+# %% ../../nbs/14_callback.schedule.ipynb 90
+@patch
+def lr_find(self:Learner, start_lr=1e-7, end_lr=10, num_it=100, stop_div=True, show_plot=True, suggest_funcs=(SuggestionMethod.Valley)):
+    "Launch a mock training to find a good learning rate and return suggestions based on `suggest_funcs` as a named tuple"
+    n_epoch = num_it//len(self.dls.train) + 1
+    cb=LRFinder(start_lr=start_lr, end_lr=end_lr, num_it=num_it, stop_div=stop_div)
+    with self.no_logging(): self.fit(n_epoch, cbs=cb)
+    if suggest_funcs is not None:
+        lrs, losses = tensor(self.recorder.lrs[num_it//10:-5]), tensor(self.recorder.losses[num_it//10:-5])
+        nan_idxs = torch.nonzero(torch.isnan(losses.view(-1)))
+        if len(nan_idxs) > 0:
+            drop_idx = min(nan_idxs)
+            lrs = lrs[:drop_idx]
+            losses = losses[:drop_idx]
+        _suggestions, nms = [], []
+        for func in tuplify(suggest_funcs):
+            nms.append(func.__name__ if not isinstance(func, partial) else func.func.__name__) # deal with partials
+            _suggestions.append(func(lrs, losses, num_it))
+        
+        SuggestedLRs = collections.namedtuple('SuggestedLRs', nms)
+        lrs, pnts = [], []
+        for lr, pnt in _suggestions:
+            lrs.append(lr)
+            pnts.append(pnt)
+        if show_plot: self.recorder.plot_lr_find(suggestions=pnts, nms=nms)
+        return SuggestedLRs(*lrs)
+
+    elif show_plot: self.recorder.plot_lr_find()

fastai/callback/tensorboard.py

@@ -0,0 +1,172 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/70a_callback.tensorboard.ipynb.
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 3
+from __future__ import annotations
+from ..basics import *
+
+# %% auto 0
+__all__ = ['TensorBoardBaseCallback', 'TensorBoardCallback', 'TensorBoardProjectorCallback', 'projector_word_embeddings',
+           'tensorboard_log']
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 18
+import tensorboard
+from torch.utils.tensorboard import SummaryWriter
+from .fp16 import ModelToHalf
+from .hook import hook_output
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 19
+class TensorBoardBaseCallback(Callback):
+    order = Recorder.order+1
+    "Base class for tensorboard callbacks"
+    def __init__(self): self.run_projector = False
+        
+    def after_pred(self):
+        if self.run_projector: self.feat = _add_projector_features(self.learn, self.h, self.feat)
+    
+    def after_validate(self):
+        if not self.run_projector: return
+        self.run_projector = False
+        self._remove()
+        _write_projector_embedding(self.learn, self.writer, self.feat)
+            
+    def after_fit(self): 
+        if self.run: self.writer.close()
+        
+    def _setup_projector(self):
+        self.run_projector = True
+        self.h = hook_output(self.learn.model[1][1] if not self.layer else self.layer)
+        self.feat = {}
+        
+    def _setup_writer(self): self.writer = SummaryWriter(log_dir=self.log_dir)
+    def __del__(self): self._remove()
+    def _remove(self):
+        if getattr(self, 'h', None): self.h.remove()
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 21
+class TensorBoardCallback(TensorBoardBaseCallback):
+    "Saves model topology, losses & metrics for tensorboard and tensorboard projector during training"
+    def __init__(self, log_dir=None, trace_model=True, log_preds=True, n_preds=9, projector=False, layer=None):
+        super().__init__()
+        store_attr()
+
+    def before_fit(self):
+        self.run = not hasattr(self.learn, 'lr_finder') and not hasattr(self, "gather_preds") and rank_distrib()==0
+        if not self.run: return
+        self._setup_writer()
+        if self.trace_model:
+            if hasattr(self.learn, 'mixed_precision'):
+                raise Exception("Can't trace model in mixed precision, pass `trace_model=False` or don't use FP16.")
+            b = self.dls.one_batch()
+            self.learn._split(b)
+            self.writer.add_graph(self.model, *self.xb)
+
+    def after_batch(self):
+        self.writer.add_scalar('train_loss', self.smooth_loss, self.train_iter)
+        for i,h in enumerate(self.opt.hypers):
+            for k,v in h.items(): self.writer.add_scalar(f'{k}_{i}', v, self.train_iter)
+
+    def after_epoch(self):
+        for n,v in zip(self.recorder.metric_names[2:-1], self.recorder.log[2:-1]):
+            self.writer.add_scalar(n, v, self.train_iter)
+        if self.log_preds:
+            b = self.dls.valid.one_batch()
+            self.learn.one_batch(0, b)
+            preds = getcallable(self.loss_func, 'activation')(self.pred)
+            out = getcallable(self.loss_func, 'decodes')(preds)
+            x,y,its,outs = self.dls.valid.show_results(b, out, show=False, max_n=self.n_preds)
+            tensorboard_log(x, y, its, outs, self.writer, self.train_iter)
+            
+    def before_validate(self):
+        if self.projector: self._setup_projector()
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 23
+class TensorBoardProjectorCallback(TensorBoardBaseCallback):
+    "Extracts and exports image featuers for tensorboard projector during inference"
+    def __init__(self, log_dir=None, layer=None):
+        super().__init__()
+        store_attr()
+    
+    def before_fit(self):
+        self.run = not hasattr(self.learn, 'lr_finder') and hasattr(self, "gather_preds") and rank_distrib()==0
+        if not self.run: return
+        self._setup_writer()
+
+    def before_validate(self):
+        self._setup_projector()
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 25
+def _write_projector_embedding(learn, writer, feat):
+    lbls = [learn.dl.vocab[l] for l in feat['lbl']] if getattr(learn.dl, 'vocab', None) else None     
+    vecs = feat['vec'].squeeze()
+    writer.add_embedding(vecs, metadata=lbls, label_img=feat['img'], global_step=learn.train_iter)
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 26
+def _add_projector_features(learn, hook, feat):
+    img = _normalize_for_projector(learn.x)
+    first_epoch = True if learn.iter == 0 else False
+    feat['vec'] = hook.stored if first_epoch else torch.cat((feat['vec'], hook.stored),0)
+    feat['img'] = img           if first_epoch else torch.cat((feat['img'], img),0)
+    if getattr(learn.dl, 'vocab', None):
+        feat['lbl'] = learn.y if first_epoch else torch.cat((feat['lbl'], learn.y),0)
+    return feat
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 27
+def _get_embeddings(model, layer):
+    layer = model[0].encoder if layer == None else layer
+    return layer.weight
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 28
+@typedispatch
+def _normalize_for_projector(x:TensorImage):
+    # normalize tensor to be between 0-1
+    img = x.clone()
+    sz = img.shape
+    img = img.view(x.size(0), -1)
+    img -= img.min(1, keepdim=True)[0]
+    img /= img.max(1, keepdim=True)[0]
+    img = img.view(*sz)
+    return img
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 29
+from ..text.all import LMLearner, TextLearner
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 30
+def projector_word_embeddings(learn=None, layer=None, vocab=None, limit=-1, start=0, log_dir=None):
+    "Extracts and exports word embeddings from language models embedding layers"
+    if not layer:
+        if   isinstance(learn, LMLearner):   layer = learn.model[0].encoder
+        elif isinstance(learn, TextLearner): layer = learn.model[0].module.encoder
+    emb = layer.weight
+    img = torch.full((len(emb),3,8,8), 0.7)
+    vocab = learn.dls.vocab[0] if vocab == None else vocab
+    vocab = list(map(lambda x: f'{x}_', vocab))
+    writer = SummaryWriter(log_dir=log_dir)
+    end = start + limit if limit >= 0 else -1
+    writer.add_embedding(emb[start:end], metadata=vocab[start:end], label_img=img[start:end])
+    writer.close()
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 32
+from ..vision.data import *
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 33
+@typedispatch
+def tensorboard_log(x:TensorImage, y: TensorCategory, samples, outs, writer, step):
+    fig,axs = get_grid(len(samples), return_fig=True)
+    for i in range(2):
+        axs = [b.show(ctx=c) for b,c in zip(samples.itemgot(i),axs)]
+    axs = [r.show(ctx=c, color='green' if b==r else 'red')
+            for b,r,c in zip(samples.itemgot(1),outs.itemgot(0),axs)]
+    writer.add_figure('Sample results', fig, step)
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 34
+from ..vision.core import TensorPoint,TensorBBox
+
+# %% ../../nbs/70a_callback.tensorboard.ipynb 35
+@typedispatch
+def tensorboard_log(x:TensorImage, y: TensorImageBase|TensorPoint|TensorBBox, samples, outs, writer, step):
+    fig,axs = get_grid(len(samples), return_fig=True, double=True)
+    for i in range(2):
+        axs[::2] = [b.show(ctx=c) for b,c in zip(samples.itemgot(i),axs[::2])]
+    for x in [samples,outs]:
+        axs[1::2] = [b.show(ctx=c) for b,c in zip(x.itemgot(0),axs[1::2])]
+    writer.add_figure('Sample results', fig, step)

fastai/callback/tracker.py

@@ -0,0 +1,139 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/17_callback.tracker.ipynb.
+
+# %% ../../nbs/17_callback.tracker.ipynb 2
+from __future__ import annotations
+from ..basics import *
+from .progress import *
+from .fp16 import MixedPrecision
+
+# %% auto 0
+__all__ = ['TerminateOnNaNCallback', 'TrackerCallback', 'EarlyStoppingCallback', 'SaveModelCallback', 'ReduceLROnPlateau']
+
+# %% ../../nbs/17_callback.tracker.ipynb 6
+class TerminateOnNaNCallback(Callback):
+    "A `Callback` that terminates training if loss is NaN."
+    order=-9
+    def after_batch(self):
+        "Test if `last_loss` is NaN and interrupts training."
+        if torch.isinf(self.loss) or torch.isnan(self.loss): raise CancelFitException
+
+# %% ../../nbs/17_callback.tracker.ipynb 10
+class TrackerCallback(Callback):
+    "A `Callback` that keeps track of the best value in `monitor`."
+    order,remove_on_fetch,_only_train_loop = 60,True,True
+    def __init__(self, 
+        monitor='valid_loss', # value (usually loss or metric) being monitored.
+        comp=None, # numpy comparison operator; np.less if monitor is loss, np.greater if monitor is metric.
+        min_delta=0., # minimum delta between the last monitor value and the best monitor value.
+        reset_on_fit=True # before model fitting, reset value being monitored to -infinity (if monitor is metric) or +infinity (if monitor is loss).
+    ):
+        if comp is None: comp = np.less if 'loss' in monitor or 'error' in monitor else np.greater
+        if comp == np.less: min_delta *= -1
+        self.monitor,self.comp,self.min_delta,self.reset_on_fit,self.best= monitor,comp,min_delta,reset_on_fit,None
+
+    def before_fit(self):
+        "Prepare the monitored value"
+        self.run = not hasattr(self, "lr_finder") and not hasattr(self, "gather_preds")
+        if self.reset_on_fit or self.best is None: self.best = float('inf') if self.comp == np.less else -float('inf')
+        assert self.monitor in self.recorder.metric_names[1:]
+        self.idx = list(self.recorder.metric_names[1:]).index(self.monitor)
+
+    def after_epoch(self):
+        "Compare the last value to the best up to now"
+        val = self.recorder.values[-1][self.idx]
+        if self.comp(val - self.min_delta, self.best): self.best,self.new_best = val,True
+        else: self.new_best = False
+
+    def after_fit(self): self.run=True
+
+# %% ../../nbs/17_callback.tracker.ipynb 19
+class EarlyStoppingCallback(TrackerCallback):
+    "A `TrackerCallback` that terminates training when monitored quantity stops improving."
+    order=TrackerCallback.order+3
+    def __init__(self, 
+        monitor='valid_loss', # value (usually loss or metric) being monitored.
+        comp=None, # numpy comparison operator; np.less if monitor is loss, np.greater if monitor is metric.
+        min_delta=0., # minimum delta between the last monitor value and the best monitor value.
+        patience=1, # number of epochs to wait when training has not improved model.
+        reset_on_fit=True # before model fitting, reset value being monitored to -infinity (if monitor is metric) or +infinity (if monitor is loss).
+    ):
+        super().__init__(monitor=monitor, comp=comp, min_delta=min_delta, reset_on_fit=reset_on_fit)
+        self.patience = patience
+
+    def before_fit(self): self.wait = 0; super().before_fit()
+    def after_epoch(self):
+        "Compare the value monitored to its best score and maybe stop training."
+        super().after_epoch()
+        if self.new_best: self.wait = 0
+        else:
+            self.wait += 1
+            if self.wait >= self.patience:
+                print(f'No improvement since epoch {self.epoch-self.wait}: early stopping')
+                raise CancelFitException()
+
+# %% ../../nbs/17_callback.tracker.ipynb 26
+class SaveModelCallback(TrackerCallback):
+    "A `TrackerCallback` that saves the model's best during training and loads it at the end."
+    order = TrackerCallback.order+1
+    def __init__(self, 
+        monitor='valid_loss', # value (usually loss or metric) being monitored.
+        comp=None, # numpy comparison operator; np.less if monitor is loss, np.greater if monitor is metric.
+        min_delta=0., # minimum delta between the last monitor value and the best monitor value.
+        fname='model', # model name to be used when saving model.
+        every_epoch=False, # if true, save model after every epoch; else save only when model is better than existing best.
+        at_end=False, # if true, save model when training ends; else load best model if there is only one saved model.
+        with_opt=False, # if true, save optimizer state (if any available) when saving model. 
+        reset_on_fit=True # before model fitting, reset value being monitored to -infinity (if monitor is metric) or +infinity (if monitor is loss).
+    ):
+        super().__init__(monitor=monitor, comp=comp, min_delta=min_delta, reset_on_fit=reset_on_fit)
+        assert not (every_epoch and at_end), "every_epoch and at_end cannot both be set to True"
+        # keep track of file path for loggers
+        self.last_saved_path = None
+        store_attr('fname,every_epoch,at_end,with_opt')
+
+    def _save(self, name): self.last_saved_path = self.learn.save(name, with_opt=self.with_opt)
+
+    def after_epoch(self):
+        "Compare the value monitored to its best score and save if best."
+        if self.every_epoch:
+            if (self.epoch%self.every_epoch) == 0: self._save(f'{self.fname}_{self.epoch}')
+        else: #every improvement
+            super().after_epoch()
+            if self.new_best:
+                print(f'Better model found at epoch {self.epoch} with {self.monitor} value: {self.best}.')
+                self._save(f'{self.fname}')
+
+    def after_fit(self, **kwargs):
+        "Load the best model."
+        if self.at_end: self._save(f'{self.fname}')
+        elif not self.every_epoch: self.learn.load(f'{self.fname}', with_opt=self.with_opt)
+
+# %% ../../nbs/17_callback.tracker.ipynb 30
+class ReduceLROnPlateau(TrackerCallback):
+    "A `TrackerCallback` that reduces learning rate when a metric has stopped improving."
+    order=TrackerCallback.order+2
+    def __init__(self, 
+        monitor='valid_loss', # value (usually loss or metric) being monitored.
+        comp=None, # numpy comparison operator; np.less if monitor is loss, np.greater if monitor is metric.
+        min_delta=0., # minimum delta between the last monitor value and the best monitor value.
+        patience=1, # number of epochs to wait when training has not improved model.
+        factor=10., # the denominator to divide the learning rate by, when reducing the learning rate.
+        min_lr=0, # the minimum learning rate allowed; learning rate cannot be reduced below this minimum.
+        reset_on_fit=True # before model fitting, reset value being monitored to -infinity (if monitor is metric) or +infinity (if monitor is loss).
+    ):
+        super().__init__(monitor=monitor, comp=comp, min_delta=min_delta, reset_on_fit=reset_on_fit)
+        self.patience,self.factor,self.min_lr = patience,factor,min_lr
+
+    def before_fit(self): self.wait = 0; super().before_fit()
+    def after_epoch(self):
+        "Compare the value monitored to its best score and reduce LR by `factor` if no improvement."
+        super().after_epoch()
+        if self.new_best: self.wait = 0
+        else:
+            self.wait += 1
+            if self.wait >= self.patience:
+                old_lr = self.opt.hypers[-1]['lr']
+                for h in self.opt.hypers: h['lr'] = max(h['lr'] / self.factor, self.min_lr)
+                self.wait = 0
+                if self.opt.hypers[-1]["lr"] < old_lr:
+                    print(f'Epoch {self.epoch}: reducing lr to {self.opt.hypers[-1]["lr"]}')