Move reading and writing of tokens into a separate package to prepare for other input and output formats

1 files changed, 490 insertions, 0 deletions

diff --git a/json_tokens/read.go b/json_tokens/read.go
new file mode 100644
index 0000000..95bbb9d
--- /dev/null
+++ b/json_tokens/read.go
@@ -0,0 +1,490 @@
+package json_tokens
+
+import (
+	"main/walk"
+	"bufio"
+	"strings"
+	"strconv"
+	"fmt"
+)
+
+type ReadAction int
+const (
+	ActionReadValue ReadAction = iota
+	ActionAppendPath
+	ActionPopPath
+	ActionIncrementPath
+	ActionAppendPathNull
+)
+
+type JSONInStructure int
+const (
+	JSONInMap JSONInStructure = iota
+	JSONInArray
+)
+
+type JSONInState int
+const (
+	JSONInValueEnd JSONInState = iota
+	JSONInValue
+	JSONInValueStart
+	JSONInString
+	JSONInKey
+)
+
+type JSONIn struct {
+	path []walk.Atom
+	reader *bufio.Reader
+	structure []JSONInStructure
+	state JSONInState
+	readBuffer []walk.Atom
+	readIndex int
+	readBufferCapacity int
+	actionBuffer []ReadAction
+	actionIndex int
+}
+
+func NewJSONIn(reader *bufio.Reader) *JSONIn {
+	return &JSONIn {
+		path: make([]walk.Atom, 0, 256),
+		reader: reader,
+		structure: []JSONInStructure{},
+		state: JSONInValueStart,
+		readBuffer: make([]walk.Atom, 0, 256),
+		readIndex: 0,
+		readBufferCapacity: 256,
+		actionBuffer: make([]ReadAction, 0, 256),
+		actionIndex: 0,
+	}
+}
+
+func isWhitespace(r rune) bool {
+	for _, ws := range " \t\r\n" {
+		if r == ws {
+			return true
+		}
+	}
+	return false
+}
+
+func isNumberRune(r rune) bool {
+	return '0' <= r && r <= '9' || r == '.'
+}
+
+func (in *JSONIn) popPath() {
+	if len(in.path) == 0 {
+		panic("Tried to pop from empty path")
+	}
+	finalAtom := in.path[len(in.path) - 1]
+	if finalAtom.Typ != walk.AtomStringTerminal {
+		in.path = in.path[:len(in.path) - 1]
+		return
+	}
+	i := len(in.path) - 2
+	for {
+		if i < 0 {
+			panic("Missing string begin in path")
+		}
+		if in.path[i].Typ == walk.AtomStringTerminal {
+			break
+		}
+		i--
+	}
+	in.path = in.path[:i]
+}
+
+func (in *JSONIn) nextNonWsRune() (rune, error) {
+	for {
+		r, _, err := in.reader.ReadRune()
+		if err != nil {
+			return 0, err
+		}
+		if !isWhitespace(r) {
+			return r, nil
+		}
+	}
+}
+
+func (in *JSONIn) requireString(criteria string) {
+	for _, r := range criteria {
+		in.require(r)
+	}
+}
+
+func (in *JSONIn) require(criterion rune) {
+	r, _, err := in.reader.ReadRune()
+	if err != nil {
+		panic("Error while reading required rune: " + err.Error())
+	}
+	if r != criterion {
+		panic("Required rune not read")
+	}
+}
+
+// Returns the first full value of a list of atoms and also a boolean to indicate if there isn't a value at the beginning
+func firstValue(atoms []walk.Atom) ([]walk.Atom, bool) {
+	if len(atoms) == 0 {
+		return nil, true
+	}
+	if atoms[0].Typ != walk.AtomStringTerminal {
+		return atoms[0:1], false
+	}
+	i := 1
+	for {
+		if i == len(atoms) {
+			return nil, true
+		}
+		if atoms[i].Typ == walk.AtomStringTerminal {
+			return atoms[0:i+1], false
+		}
+		i++
+	}
+}
+
+func (in *JSONIn) readValue() []walk.Atom {
+	try:
+	value, incomplete := firstValue(in.readBuffer[in.readIndex:])
+	if incomplete {
+		if in.readIndex == 0 {
+			newReadBuffer := make([]walk.Atom, len(in.readBuffer), in.readBufferCapacity * 2)
+			in.readBufferCapacity *= 2
+			copy(newReadBuffer, in.readBuffer)
+			in.readBuffer = newReadBuffer
+			in.fillReadBuffer()
+			goto try
+		}
+		copy(in.readBuffer, in.readBuffer[in.readIndex:])
+		in.readBuffer = in.readBuffer[:len(in.readBuffer) - in.readIndex]
+		in.readIndex = 0
+		copy(in.actionBuffer, in.actionBuffer[in.actionIndex:])
+		in.actionBuffer = in.actionBuffer[:len(in.actionBuffer) - in.actionIndex]
+		in.actionIndex = 0
+		in.fillReadBuffer()
+		goto try
+	}
+	in.readIndex += len(value)
+	return value
+}
+
+func (in *JSONIn) Read() (walk.WalkItem, error) {
+	for {
+		if in.actionIndex == len(in.actionBuffer) {
+			in.actionIndex = 0
+			in.readIndex = 0
+			in.actionBuffer = in.actionBuffer[:0]
+			in.readBuffer = in.readBuffer[:0]
+			err := in.fillReadBuffer()
+			if len(in.actionBuffer) == 0 {
+				return walk.WalkItem{}, err
+			}
+		}
+		action := in.actionBuffer[in.actionIndex]
+		in.actionIndex++
+		switch action {
+			case ActionReadValue:
+				value := in.readValue()
+				return walk.WalkItem {
+					Value: value,
+					Path: in.path,
+				}, nil
+			case ActionAppendPath:
+				value := in.readValue()
+				in.path = append(in.path, value...)
+			case ActionAppendPathNull:
+				in.path = append(in.path, walk.NewAtomNull())
+			case ActionPopPath:
+				in.popPath()
+			case ActionIncrementPath:
+				prevIndex := in.path[len(in.path) - 1]
+				if prevIndex.Typ == walk.AtomNull {
+					prevIndex = walk.NewAtomNumber(0)
+				} else if prevIndex.Typ == walk.AtomNumber {
+					prevIndex = walk.NewAtomNumber(prevIndex.Number() + 1)
+				} else {
+					panic("Invalid index in array input. Type: " + fmt.Sprintf("%v", prevIndex.Typ))
+				}
+				in.path[len(in.path) - 1] = prevIndex
+			default:
+				panic("Invalid ReadAction")
+		}
+	}
+}
+
+func (in *JSONIn) AssertDone() {
+	if len(in.structure) != 0 || in.state != JSONInValueEnd || in.readIndex < len(in.readBuffer) {
+		panic("Input ended on incomplete JSON root")
+	}
+}
+
+func (in *JSONIn) pushReadBuffer(atom walk.Atom) bool {
+	in.readBuffer = append(in.readBuffer, atom)
+	return len(in.readBuffer) == in.readBufferCapacity
+}
+
+func (in *JSONIn) pushActionBuffer(action ReadAction) {
+	in.actionBuffer = append(in.actionBuffer, action)
+}
+
+// Appends to the readBuffer until it has reached capacity
+// Also appends to the actionBuffer as needed
+func (in *JSONIn) fillReadBuffer() error {
+	switch in.state {
+		case JSONInValueStart:
+			goto valueStart
+		case JSONInValue:
+			goto value
+		case JSONInValueEnd:
+			goto valueEnd
+		case JSONInString:
+			goto string
+		case JSONInKey:
+			goto key
+		default:
+			panic("Invalid JSONInState")
+	}
+	valueStart: {
+		if len(in.structure) == 0 {
+			goto value
+		}
+		innermost := in.structure[len(in.structure) - 1]
+		switch innermost {
+			case JSONInMap:
+				goto mapValue
+			case JSONInArray:
+				goto arrayValue
+			default:
+				panic("Invalid JSONInStructure")
+		}
+	}
+	value: {
+		r, err := in.nextNonWsRune()
+		if err != nil {
+			panic("Missing value in JSON")
+		}
+		switch r {
+			case 'n':
+				in.requireString("ull")
+				in.pushActionBuffer(ActionReadValue)
+				if in.pushReadBuffer(walk.NewAtomNull()) {
+					in.state = JSONInValueEnd
+					return nil
+				}
+				goto valueEnd
+			case 'f':
+				in.requireString("alse")
+				in.pushActionBuffer(ActionReadValue)
+				if in.pushReadBuffer(walk.NewAtomBool(false)) {
+					in.state = JSONInValueEnd
+					return nil
+				}
+				goto valueEnd
+			case 't':
+				in.requireString("rue")
+				in.pushActionBuffer(ActionReadValue)
+				if in.pushReadBuffer(walk.NewAtomBool(true)) {
+					in.state = JSONInValueEnd
+					return nil
+				}
+				goto valueEnd
+			case '"':
+				in.pushActionBuffer(ActionReadValue)
+				if in.pushReadBuffer(walk.NewAtomStringTerminal()) {
+					in.state = JSONInString
+					return nil
+				}
+				goto string
+			case '{':
+				in.structure = append(in.structure, JSONInMap)
+				in.pushActionBuffer(ActionReadValue)
+				in.pushActionBuffer(ActionAppendPathNull)
+				if in.pushReadBuffer(walk.NewAtomTerminal(walk.MapBegin)) {
+					in.state = JSONInValueStart
+					return nil
+				}
+				goto mapValue
+			case '[':
+				in.structure = append(in.structure, JSONInArray)
+				in.pushActionBuffer(ActionReadValue)
+				in.pushActionBuffer(ActionAppendPathNull)
+				if in.pushReadBuffer(walk.NewAtomTerminal(walk.ArrayBegin)) {
+					in.state = JSONInValueStart
+					return nil
+				}
+				goto arrayValue
+		}
+		if isNumberRune(r) {
+			var builder strings.Builder
+			builder.WriteRune(r)
+			for {
+				r, _, err = in.reader.ReadRune()
+				if err != nil {
+					break
+				}
+				if !isNumberRune(r) {
+					in.reader.UnreadRune()
+					break
+				}
+				builder.WriteRune(r)
+			}
+			number, parseError := strconv.ParseFloat(builder.String(), 64)
+			if parseError != nil {
+				panic("Invalid number")
+			}
+			in.pushActionBuffer(ActionReadValue)
+			if in.pushReadBuffer(walk.NewAtomNumber(number)) {
+				in.state = JSONInValueEnd
+				return nil
+			}
+			goto valueEnd
+		}
+		panic("Invalid JSON value starting with: " + string(r))
+	}
+	string: {
+		r, _, err := in.reader.ReadRune()
+		if err != nil {
+			panic("Missing closing terminal in string input: " + err.Error())
+		}
+		if r == '"' {
+			if in.pushReadBuffer(walk.NewAtomStringTerminal()) {
+				in.state = JSONInValueEnd
+				return nil
+			}
+			goto valueEnd
+		}
+		if r == '\\' {
+			r, _, err = in.reader.ReadRune()
+			if err != nil {
+				panic("Missing rune after \\")
+			}
+			if in.pushReadBuffer(walk.NewAtomStringRune(r)) {
+				in.state = JSONInString
+				return nil
+			}
+			goto string
+		}
+		if in.pushReadBuffer(walk.NewAtomStringRune(r)) {
+			in.state = JSONInString
+			return nil
+		}
+		goto string
+	}
+	key: {
+		var full bool
+		for {
+			r, _, err := in.reader.ReadRune()
+			if err != nil {
+				panic("Missing closing terminal in string input: " + err.Error())
+			}
+			if r == '"' {
+				full = in.pushReadBuffer(walk.NewAtomStringTerminal())
+				break
+			}
+			if r == '\\' {
+				r, _, err = in.reader.ReadRune()
+				if err != nil {
+					panic("Missing rune after \\")
+				}
+				if in.pushReadBuffer(walk.NewAtomStringRune(r)) {
+					in.state = JSONInKey
+					return nil
+				}
+				continue
+			}
+			if in.pushReadBuffer(walk.NewAtomStringRune(r)) {
+				in.state = JSONInKey
+				return nil
+			}
+			continue
+		}
+		r, err := in.nextNonWsRune()
+		if err != nil {
+			panic("Expected : got: " + err.Error())
+		}
+		if r != ':' {
+			panic("Expected : after key")
+		}
+		if full {
+			in.state = JSONInValue
+			return nil
+		}
+		goto value
+	}
+	valueEnd: {
+		r, err := in.nextNonWsRune()
+		if err != nil {
+			in.state = JSONInValueEnd
+			return err
+		}
+		if len(in.structure) == 0 {
+			panic("More input after root JSON object ends")
+		}
+		innermost := in.structure[len(in.structure) - 1]
+		if innermost == JSONInMap && r == '}' {
+			in.structure = in.structure[:len(in.structure) - 1]
+			in.pushActionBuffer(ActionPopPath)
+			in.pushActionBuffer(ActionReadValue)
+			if in.pushReadBuffer(walk.NewAtomTerminal(walk.MapEnd)) {
+				in.state = JSONInValueEnd
+				return nil
+			}
+			goto valueEnd
+		} else if innermost == JSONInArray && r == ']' {
+			in.structure = in.structure[:len(in.structure) - 1]
+			in.pushActionBuffer(ActionPopPath)
+			in.pushActionBuffer(ActionReadValue)
+			if in.pushReadBuffer(walk.NewAtomTerminal(walk.ArrayEnd)) {
+				in.state = JSONInValueEnd
+				return nil
+			}
+			goto valueEnd
+		}
+		if r != ',' {
+			panic("Expected , after JSON value, found: \"" + string(r) + "\"")
+		}
+		goto valueStart
+	}
+	mapValue: {
+		in.pushActionBuffer(ActionPopPath)
+		r, err := in.nextNonWsRune()
+		if err != nil {
+			panic("Missing value inside object")
+		}
+		if r == '}' {
+			in.structure = in.structure[:len(in.structure) - 1]
+			in.pushActionBuffer(ActionReadValue)
+			if in.pushReadBuffer(walk.NewAtomTerminal(walk.MapEnd)) {
+				in.state = JSONInValueEnd
+				return nil
+			}
+			goto valueEnd
+		}
+		if r != '"' {
+			panic("Expected key found something else")
+		}
+		in.pushActionBuffer(ActionAppendPath)
+		if in.pushReadBuffer(walk.NewAtomStringTerminal()) {
+			in.state = JSONInKey
+			return nil
+		}
+		goto key
+	}
+	arrayValue: {
+		r, err := in.nextNonWsRune()
+		if err != nil {
+			panic("Missing value inside array")
+		}
+		if r == ']' {
+			in.structure = in.structure[:len(in.structure) - 1]
+			in.pushActionBuffer(ActionPopPath)
+			in.pushActionBuffer(ActionReadValue)
+			if in.pushReadBuffer(walk.NewAtomTerminal(walk.ArrayEnd)) {
+				in.state = JSONInValueEnd
+				return nil
+			}
+			goto valueEnd
+		}
+		in.reader.UnreadRune()
+		in.pushActionBuffer(ActionIncrementPath)
+		goto value
+	}
+}