Class new title: TokenCollector
	subclassof: Object
	fields: 'sink parenstack'
	declare: ''

comment: t1 
	[]

contents 
	[until parenstack empty do
		[self rightparen].
	sink contents]

default 
	[self to: (Vector new: 20)]

float: t1 fraction: t2 exp: t3 
	[self next_ (t1 + '.' + t2 + 'e' + t3) asFloat]

identifier: t1 
	[self next_ t1 unique]

integer: t1 
	[self next_ t1 asInteger]

keyword: t1 
	[self next_ t1 unique]

leftparen 
	[parenstack next_ sink.
	sink _ (Vector new: 10) asStream]

next_ t1 
	[sink next_ t1]

notify: t1 
	[user notify: t1]

onechar: t1 | t2 
	[t2 _ String new: 1.
	t2  1 _ t1.
	self next_ t2 unique]

otheratom: t1 
	[self next_ t1 unique]

rightparen 
	[parenstack empty  []
	parenstack last next_ sink contents.
	sink _ parenstack pop]

separator: t1 
	[]

string: t1 
	[self next_ t1]

to: t1 
	[sink _ t1 asStream.
	parenstack _ (Vector new: 5) asStream]

trailer: t1 
	[]