Package io.github.kirstenali.deepj.data

Class RandomAccessTextDataset

java.lang.Object

io.github.kirstenali.deepj.data.RandomAccessTextDataset

All Implemented Interfaces:

BatchSource, StatefulBatchSource, StatefulTrainingDataset, AutoCloseable

public final class RandomAccessTextDataset
extends Object
implements StatefulTrainingDataset

Constructor Summary

Constructors

Constructor	Description
RandomAccessTextDataset(Path path, Tokenizer tokenizer, int seqLen, long seed)
RandomAccessTextDataset(Path path, Tokenizer tokenizer, int seqLen, long seed, TextFileRange range)

Method Summary

Modifier and Type	Method	Description
void	close()
Batch	nextBatch(int batchSize)
long	randomState()
void	restoreRandomState(long state)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Details

RandomAccessTextDataset

public RandomAccessTextDataset(Path path,
 Tokenizer tokenizer,
 int seqLen,
 long seed)
                        throws IOException

Throws:

IOException

RandomAccessTextDataset

public RandomAccessTextDataset(Path path,
 Tokenizer tokenizer,
 int seqLen,
 long seed,
 TextFileRange range)
                        throws IOException

Throws:

IOException

Method Details

nextBatch

public Batch nextBatch(int batchSize)

Specified by:

nextBatch in interface BatchSource

randomState

public long randomState()

Specified by:

randomState in interface StatefulBatchSource

restoreRandomState

public void restoreRandomState(long state)

Specified by:

restoreRandomState in interface StatefulBatchSource

close

public void close()
           throws IOException

Specified by:

close in interface AutoCloseable

Throws:

IOException