= np.random.randint(0, 2, size=(10)) # Dataset of size 10 (train=8, valid=2)
lbls = lambda i: i >= 8
is_valid = DataBlock(blocks=[CategoryBlock],
dblock =[lambda i: lbls[i]], splitter=FuncSplitter(is_valid))
getters= dblock.datasets(list(range(10)))
dset = [ToTensor()]
item_tfms = range(8) # len(wgts) == 8
wgts = dset.weighted_dataloaders(bs=1, wgts=wgts, after_item=item_tfms) dls
Data Callbacks
Callbacks which work with a learner’s data
CollectDataCallback
CollectDataCallback (after_create=None, before_fit=None, before_epoch=None, before_train=None, before_batch=None, after_pred=None, after_loss=None, before_backward=None, after_cancel_backward=None, after_backward=None, before_step=None, after_cancel_step=None, after_step=None, after_cancel_batch=None, after_batch=None, after_cancel_train=None, after_train=None, before_validate=None, after_cancel_validate=None, after_validate=None, after_cancel_epoch=None, after_epoch=None, after_cancel_fit=None, after_fit=None)
Collect all batches, along with pred
and loss
, into self.data
. Mainly for testing
WeightedDL
WeightedDL (dataset=None, bs=None, wgts=None, shuffle:bool=False, num_workers:int=None, verbose:bool=False, do_setup:bool=True, pin_memory=False, timeout=0, batch_size=None, drop_last=False, indexed=None, n=None, device=None, persistent_workers=False, pin_memory_device='', wif=None, before_iter=None, after_item=None, before_batch=None, after_batch=None, after_iter=None, create_batches=None, create_item=None, create_batch=None, retain=None, get_idxs=None, sample=None, shuffle_fn=None, do_batch=None)
Weighted dataloader where wgts
is used for the training set only
Datasets.weighted_dataloaders
Datasets.weighted_dataloaders (wgts, bs=64, shuffle_train:bool=None, shuffle:bool=True, val_shuffle:bool=False, n:int=None, path:str|Path='.', dl_type:TfmdDL=None, dl_kwargs:list=None, device:torch.device=None, drop_last:bool=None, val_bs:int=None)
Create a weighted dataloader WeightedDL
with wgts
for the training set
Type | Default | Details | |
---|---|---|---|
wgts | |||
bs | int | 64 | Batch size |
shuffle_train | bool | None | (Deprecated, use shuffle ) Shuffle training DataLoader |
shuffle | bool | True | Shuffle training DataLoader |
val_shuffle | bool | False | Shuffle validation DataLoader |
n | int | None | Size of Datasets used to create DataLoader |
path | str | pathlib.Path | . | Path to put in DataLoaders |
dl_type | TfmdDL | None | Type of DataLoader |
dl_kwargs | list | None | List of kwargs to pass to individual DataLoader s |
device | device | None | Device to put DataLoaders |
drop_last | bool | None | Drop last incomplete batch, defaults to shuffle |
val_bs | int | None | Validation batch size, defaults to bs |
# if len(wgts) != 8, this will fail" dls.show_batch()
1
= 160
n = Datasets(torch.arange(n).float())
dsets = dsets.weighted_dataloaders(wgts=range(n), bs=16)
dls = synth_learner(data=dls, cbs=CollectDataCallback) learn
1)
learn.fit(= concat(*learn.collect_data.data.itemgot(0,0))
t ; plt.hist(t.numpy())
[0, nan, None, '00:00']
DataBlock.weighted_dataloaders
DataBlock.weighted_dataloaders (source, wgts, bs=64, verbose:bool=False, shuffle_train:bool=None, shuffle:bool=True, val_shuffle:bool=False, n:int=None, path:str|Path='.', dl_type:TfmdDL=None, dl_kwargs:list=None, device:torch.device=None, drop_last:bool=None, val_bs:int=None)
Create a weighted dataloader WeightedDL
with wgts
for the dataset
Type | Default | Details | |
---|---|---|---|
source | |||
wgts | |||
bs | int | 64 | Batch size |
verbose | bool | False | |
shuffle_train | bool | None | (Deprecated, use shuffle ) Shuffle training DataLoader |
shuffle | bool | True | Shuffle training DataLoader |
val_shuffle | bool | False | Shuffle validation DataLoader |
n | int | None | Size of Datasets used to create DataLoader |
path | str | pathlib.Path | . | Path to put in DataLoaders |
dl_type | TfmdDL | None | Type of DataLoader |
dl_kwargs | list | None | List of kwargs to pass to individual DataLoader s |
device | device | None | Device to put DataLoaders |
drop_last | bool | None | Drop last incomplete batch, defaults to shuffle |
val_bs | int | None | Validation batch size, defaults to bs |
= dblock.weighted_dataloaders(list(range(10)), wgts, bs=1)
dls dls.show_batch()
0
PartialDL
PartialDL (dataset=None, bs=None, partial_n=None, shuffle:bool=False, num_workers:int=None, verbose:bool=False, do_setup:bool=True, pin_memory=False, timeout=0, batch_size=None, drop_last=False, indexed=None, n=None, device=None, persistent_workers=False, pin_memory_device='', wif=None, before_iter=None, after_item=None, before_batch=None, after_batch=None, after_iter=None, create_batches=None, create_item=None, create_batch=None, retain=None, get_idxs=None, sample=None, shuffle_fn=None, do_batch=None)
Select randomly partial quantity of data at each epoch
FilteredBase.partial_dataloaders
FilteredBase.partial_dataloaders (partial_n, bs=64, shuffle_train:bool=None, shuffle:bool=True, val_shuffle:bool=False, n:int=None, path:str|Path='.', dl_type:TfmdDL=None, dl_kwargs:list=None, device:torch.device=None, drop_last:bool=None, val_bs:int=None)
Create a partial dataloader PartialDL
for the training set
Type | Default | Details | |
---|---|---|---|
partial_n | |||
bs | int | 64 | Batch size |
shuffle_train | bool | None | (Deprecated, use shuffle ) Shuffle training DataLoader |
shuffle | bool | True | Shuffle training DataLoader |
val_shuffle | bool | False | Shuffle validation DataLoader |
n | int | None | Size of Datasets used to create DataLoader |
path | str | pathlib.Path | . | Path to put in DataLoaders |
dl_type | TfmdDL | None | Type of DataLoader |
dl_kwargs | list | None | List of kwargs to pass to individual DataLoader s |
device | device | None | Device to put DataLoaders |
drop_last | bool | None | Drop last incomplete batch, defaults to shuffle |
val_bs | int | None | Validation batch size, defaults to bs |
= dsets.partial_dataloaders(partial_n=32, bs=16) dls
assert len(dls[0])==2
for batch in dls[0]:
assert len(batch[0])==16