32 lines
706 B
Go
32 lines
706 B
Go
package schema
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestTokenize(t *testing.T) {
|
|
in := "(test ; test comment\n" +
|
|
"@test) ; test comment\n" +
|
|
`(test "Hello World")` + "\n" +
|
|
"; test comment 2\n" +
|
|
"(+ 1 2)\n" +
|
|
"(test `\"Hello world\"`)\n"
|
|
want := "[(0][n'test'][n'@test'][0)]\n" +
|
|
"[(0][n'test'][l'Hello World'][0)]\n" +
|
|
"[(0][n'+'][l1][l2][0)]\n" +
|
|
"[(0][n'test'][l'\"Hello world\"'][0)]\n"
|
|
tokens, _ := Tokenize([]byte(in))
|
|
var test strings.Builder
|
|
|
|
for _, statement := range tokens {
|
|
for _, token := range statement {
|
|
test.WriteString(token.String())
|
|
}
|
|
test.WriteString("\n")
|
|
}
|
|
if test.String() != want {
|
|
t.Errorf("\ngot:\n%s\nwant:\n%s", test.String(), want)
|
|
}
|
|
}
|