ExternalTokenizer

class ExternalTokenizer(tokenFn: (input: InputStream, stack: Stack) -> Unit, val contextual: Boolean = false, val fallback: Boolean = false, val extend: Boolean = false) : Tokenizer

@external tokens declarations in the grammar should resolve to an instance of this class.

Constructors

constructor(tokenFn: (input: InputStream, stack: Stack) -> Unit, contextual: Boolean = false, fallback: Boolean = false, extend: Boolean = false)

open override val contextual: Boolean

open override val extend: Boolean

open override val fallback: Boolean

open override fun token(input: InputStream, stack: Stack)