initial commit

schema/token_test.go

@@ -0,0 +1,31 @@
+package schema
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestTokenize(t *testing.T) {
+	in := "(test ; test comment\n" +
+		"@test) ; test comment\n" +
+		`(test "Hello World")` + "\n" +
+		"; test comment 2\n" +
+		"(+ 1 2)\n" +
+		"(test `\"Hello world\"`)\n"
+	want := "[(0][n'test'][n'@test'][0)]\n" +
+		"[(0][n'test'][l'Hello World'][0)]\n" +
+		"[(0][n'+'][l1][l2][0)]\n" +
+		"[(0][n'test'][l'\"Hello world\"'][0)]\n"
+	tokens, _ := Tokenize([]byte(in))
+	var test strings.Builder
+
+	for _, statement := range tokens {
+		for _, token := range statement {
+			test.WriteString(token.String())
+		}
+		test.WriteString("\n")
+	}
+	if test.String() != want {
+		t.Errorf("\ngot:\n%s\nwant:\n%s", test.String(), want)
+	}
+}