# Text core
## Preprocessing rules
The following are rules applied to texts before or after it’s tokenized.
------------------------------------------------------------------------
source
### spec_add_spaces
``` python
def spec_add_spaces(
t
):
```
*Add spaces around / and \#*
``` python
test_eq(spec_add_spaces('#fastai'), ' # fastai')
test_eq(spec_add_spaces('/fastai'), ' / fastai')
test_eq(spec_add_spaces('\\fastai'), ' \\ fastai')
```
------------------------------------------------------------------------
source
### rm_useless_spaces
``` python
def rm_useless_spaces(
t
):
```
*Remove multiple spaces*
``` python
test_eq(rm_useless_spaces('a b c'), 'a b c')
```
------------------------------------------------------------------------
source
### replace_rep
``` python
def replace_rep(
t
):
```
*Replace repetitions at the character level: cccc – TK_REP 4 c*
It starts replacing at 3 repetitions of the same character or more.
``` python
test_eq(replace_rep('aa'), 'aa')
test_eq(replace_rep('aaaa'), f' {TK_REP} 4 a ')
```
------------------------------------------------------------------------
source
### replace_wrep
``` python
def replace_wrep(
t
):
```
*Replace word repetitions: word word word word – TK_WREP 4 word*
It starts replacing at 3 repetitions of the same word or more.
``` python
test_eq(replace_wrep('ah ah'), 'ah ah')
test_eq(replace_wrep('ah ah ah'), f' {TK_WREP} 3 ah ')
test_eq(replace_wrep('ah ah ah ah'), f' {TK_WREP} 4 ah ')
test_eq(replace_wrep('ah ah ah ah '), f' {TK_WREP} 4 ah ')
test_eq(replace_wrep('ah ah ah ah.'), f' {TK_WREP} 4 ah .')
test_eq(replace_wrep('ah ah ahi'), f'ah ah ahi')
```
------------------------------------------------------------------------
source
### fix_html
``` python
def fix_html(
x
):
```
*Various messy things we’ve seen in documents*
``` python
test_eq(fix_html('#39;bli#146;'), "'bli'")
test_eq(fix_html('Sarah amp; Duck...'), 'Sarah & Duck …')
test_eq(fix_html('a nbsp; #36;'), 'a $')
test_eq(fix_html('\\" '), f'" {UNK}')
test_eq(fix_html('quot; @.@ @-@ '), "' .-")
test_eq(fix_html('
text\\n'), '\ntext\n')
```
------------------------------------------------------------------------
source
### replace_all_caps
``` python
def replace_all_caps(
t
):
```
*Replace tokens in ALL CAPS by their lower version and add `TK_UP`
before.*
``` python
test_eq(replace_all_caps("I'M SHOUTING"), f"{TK_UP} i'm {TK_UP} shouting")
test_eq(replace_all_caps("I'm speaking normally"), "I'm speaking normally")
test_eq(replace_all_caps("I am speaking normally"), "i am speaking normally")
```
------------------------------------------------------------------------
source
### replace_maj
``` python
def replace_maj(
t
):
```
*Replace tokens in Sentence Case by their lower version and add `TK_MAJ`
before.*
``` python
test_eq(replace_maj("Jeremy Howard"), f'{TK_MAJ} jeremy {TK_MAJ} howard')
test_eq(replace_maj("I don't think there is any maj here"), ("i don't think there is any maj here"),)
```
------------------------------------------------------------------------
source
### lowercase
``` python
def lowercase(
t, add_bos:bool=True, add_eos:bool=False
):
```
*Converts `t` to lowercase*
------------------------------------------------------------------------
source
### replace_space
``` python
def replace_space(
t
):
```
*Replace embedded spaces in a token with unicode line char to allow for
split/join*
## Tokenizing
A tokenizer is a class that must implement `__call__`. This method
receives a iterator of texts and must return a generator with their
tokenized versions. Here is the most basic example:
------------------------------------------------------------------------
source
### BaseTokenizer
``` python
def BaseTokenizer(
split_char:str=' ', **kwargs
):
```
*Basic tokenizer that just splits on spaces*
``` python
tok = BaseTokenizer()
test_eq(tok(["This is a text"]), [["This", "is", "a", "text"]])
tok = BaseTokenizer('x')
test_eq(tok(["This is a text"]), [["This is a te", "t"]])
```
------------------------------------------------------------------------
source
### SpacyTokenizer
``` python
def SpacyTokenizer(
lang:str='en', special_toks:NoneType=None, buf_sz:int=5000
):
```
*Spacy tokenizer for `lang`*
``` python
tok = SpacyTokenizer()
inp,exp = "This isn't the easiest text.",["This", "is", "n't", "the", "easiest", "text", "."]
test_eq(L(tok([inp,inp])), [exp,exp])
```
------------------------------------------------------------------------
source
### TokenizeWithRules
``` python
def TokenizeWithRules(
tok, rules:NoneType=None, post_rules:NoneType=None
):
```
*A wrapper around `tok` which applies `rules`, then tokenizes, then
applies `post_rules`*
``` python
f = TokenizeWithRules(BaseTokenizer(),rules=[replace_all_caps])
test_eq(f(["THIS isn't a problem"]), [[TK_UP, 'this', "isn't", 'a', 'problem']])
f = TokenizeWithRules(SpacyTokenizer())
test_eq(f(["This isn't a problem"]), [[BOS, TK_MAJ, 'this', 'is', "n't", 'a', 'problem']])
f = TokenizeWithRules(BaseTokenizer(split_char="'"), rules=[])
test_eq(f(["This isn't a problem"]), [['This▁isn', 't▁a▁problem']])
```
The main function that will be called during one of the processes
handling tokenization. It will iterate through the `batch` of texts,
apply them `rules` and tokenize them.
``` python
texts = ["this is a text", "this is another text"]
tok = TokenizeWithRules(BaseTokenizer(), texts.__getitem__)
test_eq(tok([0,1]), [['this', 'is', 'a', 'text'],['this', 'is', 'another', 'text']])
```
------------------------------------------------------------------------
source
### tokenize1
``` python
def tokenize1(
text, tok, rules:NoneType=None, post_rules:NoneType=None
):
```
*Call
[`TokenizeWithRules`](https://docs.fast.ai/text.core.html#tokenizewithrules)
with a single text*
``` python
test_eq(tokenize1("This isn't a problem", SpacyTokenizer()),
[BOS, TK_MAJ, 'this', 'is', "n't", 'a', 'problem'])
test_eq(tokenize1("This isn't a problem", tok=BaseTokenizer(), rules=[]),
['This',"isn't",'a','problem'])
```
To run your own stateful transform this way - one instance per worker
process, results streamed back with their indices - use
`ProcessPoolExecutor` with an `initializer`, as
[`parallel_tokenize`](https://docs.fast.ai/text.core.html#parallel_tokenize)
does. [`_pg_setup`](https://docs.fast.ai/text.core.html#_pg_setup)
builds the per-process instance, and
[`_pg_call`](https://docs.fast.ai/text.core.html#_pg_call) runs a batch
through it. For instance:
``` python
class TestSleepyBatchFunc:
"For testing parallel processes that run at different speeds"
def __init__(self): self.a=1
def __call__(self, batch):
for k in batch:
time.sleep(random.random()/10)
yield k+self.a
x = np.linspace(0,0.99,20)
if parallelable('n_workers', 2):
batches = L(chunked(x, n_chunks=2))
idxs = L(itertools.accumulate(0 + batches.map(len)))
with ProcessPoolExecutor(2, initializer=_pg_setup, initargs=(TestSleepyBatchFunc, {})) as ex:
futs = {ex.submit(_pg_call, b): st for b,st in zip(batches, idxs)}
res = L((i,v) for f in as_completed(futs) for i,v in enumerate(f.result(), futs[f]))
test_eq(res.sorted().itemgot(1), x+1)
```
------------------------------------------------------------------------
source
### parallel_tokenize
``` python
def parallel_tokenize(
items, tok:NoneType=None, rules:NoneType=None, n_workers:int=4, progress:bool=False, **kwargs
):
```
*Call optional `setup` on `tok`, then tokenize `items` with
[`TokenizeWithRules`](https://docs.fast.ai/text.core.html#tokenizewithrules)
in parallel, yielding unordered `(idx,toks)` pairs*
Note that the generator returned contains tuples of indices and results.
There is no guarantee that the results are returned in order, so you
should sort by the first item of the tuples (the indices) if you need
them ordered.
``` python
res = parallel_tokenize(['0 1', '1 2'], rules=[], n_workers=2)
idxs,toks = zip(*L(res).sorted(itemgetter(0)))
test_eq(toks, [['0','1'],['1','2']])
```
### Tokenize texts in files
Preprocessing function for texts in filenames. Tokenized texts will be
saved in a similar fashion in a directory suffixed with `_tok` in the
parent folder of `path` (override with `output_dir`). This directory is
the return value.
------------------------------------------------------------------------
source
### tokenize_folder
``` python
def tokenize_folder(
path, extensions:NoneType=None, folders:NoneType=None, output_dir:NoneType=None, skip_if_exists:bool=True,
output_names:NoneType=None, n_workers:int=4, rules:NoneType=None, tok:NoneType=None, encoding:str='utf8'
):
```
*Tokenize text files in `path` in parallel using `n_workers`*
The result will be in `output_dir` (defaults to a folder in the same
parent directory as `path`, with `_tok` added to `path.name`) with the
same structure as in `path`. Tokenized texts for a given file will be in
the file having the same name in `output_dir`. Additionally, a file with
a .len suffix contains the number of tokens and the count of all words
is stored in `output_dir/counter.pkl`.
`extensions` will default to `['.txt']` and all text files in `path` are
treated unless you specify a list of folders in `include`. `rules` (that
defaults to `defaults.text_proc_rules`) are applied to each text before
going in the tokenizer.
------------------------------------------------------------------------
source
### tokenize_files
``` python
def tokenize_files(
files, path, output_dir, output_names:NoneType=None, n_workers:int=4, rules:NoneType=None, tok:NoneType=None,
encoding:str='utf8', skip_if_exists:bool=False
):
```
*Tokenize text `files` in parallel using `n_workers`*
### Tokenize texts in a dataframe
------------------------------------------------------------------------
source
### tokenize_texts
``` python
def tokenize_texts(
texts, n_workers:int=4, rules:NoneType=None, tok:NoneType=None
):
```
*Tokenize `texts` in parallel using `n_workers`*
------------------------------------------------------------------------
source
### tokenize_df
``` python
def tokenize_df(
df, text_cols, n_workers:int=4, rules:NoneType=None, mark_fields:NoneType=None, tok:NoneType=None,
tok_text_col:str='text'
):
```
*Tokenize texts in `df[text_cols]` in parallel using `n_workers` and
stores them in `df[tok_text_col]`*
This function returns a new dataframe with the same non-text columns, a
column named text that contains the tokenized texts and a column named
text_lengths that contains their respective length. It also returns a
counter of all seen words to quickly build a vocabulary afterward.
`rules` (that defaults to `defaults.text_proc_rules`) are applied to
each text before going in the tokenizer. If `mark_fields` isn’t
specified, it defaults to `False` when there is a single text column,
`True` when there are several. In that case, the texts in each of those
columns are joined with `FLD` markers followed by the number of the
field.
------------------------------------------------------------------------
source
### tokenize_csv
``` python
def tokenize_csv(
fname, text_cols, outname:NoneType=None, n_workers:int=4, rules:NoneType=None, mark_fields:NoneType=None,
tok:NoneType=None, header:str='infer', chunksize:int=50000
):
```
*Tokenize texts in the `text_cols` of the csv `fname` in parallel using
`n_workers`*
------------------------------------------------------------------------
source
### load_tokenized_csv
``` python
def load_tokenized_csv(
fname
):
```
*Utility function to quickly load a tokenized csv ans the corresponding
counter*
The result will be written in a new csv file in `outname` (defaults to
the same as `fname` with the suffix `_tok.csv`) and will have the same
header as the original file, the same non-text columns, a text and a
text_lengths column as described in
[`tokenize_df`](https://docs.fast.ai/text.core.html#tokenize_df).
`rules` (that defaults to `defaults.text_proc_rules`) are applied to
each text before going in the tokenizer. If `mark_fields` isn’t
specified, it defaults to `False` when there is a single text column,
`True` when there are several. In that case, the texts in each of those
columns are joined with `FLD` markers followed by the number of the
field.
The csv file is opened with `header` and optionally with blocks of
`chunksize` at a time. If this argument is passed, each chunk is
processed independently and saved in the output file to save memory
usage.
``` python
def _prepare_texts(tmp_d):
"Prepare texts in a folder struct in tmp_d, a csv file and returns a dataframe"
path = Path(tmp_d)/'tmp'
path.mkdir()
for d in ['a', 'b', 'c']:
(path/d).mkdir()
for i in range(5):
with open(path/d/f'text{i}.txt', 'w') as f: f.write(f"This is an example of text {d} {i}")
texts = [f"This is an example of text {d} {i}" for i in range(5) for d in ['a', 'b', 'c']]
df = pd.DataFrame({'text': texts, 'label': list(range(15))}, columns=['text', 'label'])
csv_fname = tmp_d/'input.csv'
df.to_csv(csv_fname, index=False)
return path,df,csv_fname
```
## [`Tokenizer`](https://docs.fast.ai/text.core.html#tokenizer)-
------------------------------------------------------------------------
source
### Tokenizer
``` python
def Tokenizer(
tok, rules:NoneType=None, counter:NoneType=None, lengths:NoneType=None, mode:NoneType=None, sep:str=' '
):
```
*Provides a consistent `Transform` interface to tokenizers operating on
`DataFrame`s and folders*
``` python
with tempfile.TemporaryDirectory() as tmp_d:
path,df,csv_fname = _prepare_texts(Path(tmp_d))
items = get_text_files(path)
splits = RandomSplitter()(items)
dsets = Datasets(items, [Tokenizer.from_folder(path)], splits=splits)
print(dsets.train[0])
dsets = Datasets(df, [Tokenizer.from_df('text')], splits=splits)
print(dsets.train[0][0].text)
```
(['xxbos', 'xxmaj', 'this', 'is', 'an', 'example', 'of', 'text', 'b', '0'],)
('xxbos', 'xxmaj', 'this', 'is', 'an', 'example', 'of', 'text', 'c', '3')
``` python
tst = test_set(dsets, ['This is a test', 'this is another test'])
test_eq(tst, [(['xxbos', 'xxmaj', 'this','is','a','test'],),
(['xxbos','this','is','another','test'],)])
```
## Sentencepiece
------------------------------------------------------------------------
source
### SentencePieceTokenizer
``` python
def SentencePieceTokenizer(
lang:str='en', special_toks:NoneType=None, sp_model:NoneType=None, vocab_sz:NoneType=None,
max_vocab_sz:int=30000, model_type:str='unigram', char_coverage:NoneType=None, cache_dir:str='tmp'
):
```
*SentencePiece tokenizer for `lang`*
``` python
texts = [f"This is an example of text {i}" for i in range(10)]
df = pd.DataFrame({'text': texts, 'label': list(range(10))}, columns=['text', 'label'])
out,cnt = tokenize_df(df, text_cols='text', tok=SentencePieceTokenizer(vocab_sz=34), n_workers=1)
```
``` python
with tempfile.TemporaryDirectory() as tmp_d:
path,df,csv_fname = _prepare_texts(Path(tmp_d))
items = get_text_files(path)
splits = RandomSplitter()(items)
tok = SentencePieceTokenizer(special_toks=[])
dsets = Datasets(items, [Tokenizer.from_folder(path, tok=tok)], splits=splits)
print(dsets.train[0][0])
with warnings.catch_warnings():
dsets = Datasets(df, [Tokenizer.from_df('text', tok=tok)], splits=splits)
print(dsets.train[0][0].text)
```
['▁xx', 'b', 'o', 's', '▁xx', 'm', 'a', 'j', '▁t', 'h', 'i', 's', '▁', 'i', 's', '▁a', 'n', '▁', 'ex', 'a', 'm', 'p', 'l', 'e', '▁', 'o', 'f', '▁t', 'ex', 't', '▁', 'b', '▁', '2']
['▁xx', 'b', 'o', 's', '▁xx', 'm', 'a', 'j', '▁t', 'h', 'i', 's', '▁', 'i', 's', '▁a', 'n', '▁', 'ex', 'a', 'm', 'p', 'l', 'e', '▁', 'o', 'f', '▁t', 'ex', 't', '▁a', '▁', '4']
/home/jhoward/miniconda3/lib/python3.8/site-packages/numpy/core/_asarray.py:102: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.
return array(a, dtype, copy=False, order=order)