Modify subex to take JSON split into "data"

Currently no way to reassemble the data on the other side
Much of the potential data cannot be interacted with meaningfully, only the string functionality is implemented
Should rename data to something else

1 files changed, 49 insertions, 34 deletions

diff --git a/subex/subexstate.go b/subex/subexstate.go
index 2e613e8..9e0d61a 100644
--- a/subex/subexstate.go
+++ b/subex/subexstate.go
@@ -1,35 +1,41 @@
 package subex
 
 import (
-	"strings"
+	"main/walk"
 )
 
+// A state of execution for the transducer
 type SubexState interface {
-	eat(store Store, char rune) []SubexBranch
-	accepting(store Store) []string
+	// Eat a datum and transition to any number of new states
+	eat(store Store, char walk.Datum) []SubexBranch
+	// Find accepting states reachable through epsilon transitions and return their outputs
+	accepting(store Store) [][]walk.Datum
 }
 
+// Try first, if it fails then try second
 type SubexGroupState struct {
 	first, second SubexState
 }
-func (state SubexGroupState) eat(store Store, char rune) []SubexBranch {
+func (state SubexGroupState) eat(store Store, char walk.Datum) []SubexBranch {
 	otherStore := store.clone()
 	return append(state.first.eat(store, char), state.second.eat(otherStore, char)...)
 }
-func (state SubexGroupState) accepting(store Store) []string {
+func (state SubexGroupState) accepting(store Store) [][]walk.Datum {
 	return append(state.first.accepting(store), state.second.accepting(store)...)
 }
 
+// Run the match machine and store the output in a slot for later use
+// Output nothing
 type SubexStoreState struct {
 	match SubexState
 	slot rune
 	next SubexState
-	toStore string
+	toStore []walk.Datum
 }
-func (state SubexStoreState) eat(store Store, char rune) (nextStates []SubexBranch) {
+func (state SubexStoreState) eat(store Store, char walk.Datum) (nextStates []SubexBranch) {
 	acceptedOutputs := state.match.accepting(store)
 	for _, acceptedOutput := range acceptedOutputs {
-		nextStore := store.withValue(state.slot, state.toStore + acceptedOutput)
+		nextStore := store.withValue(state.slot, append(state.toStore, acceptedOutput...))
 		nextStates = append(nextStates, state.next.eat(nextStore.clone(), char)...)
 	}
 	nextMatchStates := state.match.eat(store.clone(), char)
@@ -39,107 +45,116 @@ func (state SubexStoreState) eat(store Store, char rune) (nextStates []SubexBran
 				match: matchState.state,
 				slot: state.slot,
 				next: state.next,
-				toStore: state.toStore + matchState.output,
+				toStore: append(state.toStore, matchState.output...),
 			},
-			output: "",
+			output: nil,
 			store: store.clone(),
 		})
 	}
 	return nextStates
 }
-func (state SubexStoreState) accepting(store Store) (outputs []string) {
+func (state SubexStoreState) accepting(store Store) (outputs [][]walk.Datum) {
 	acceptedOutputs := state.match.accepting(store)
 	for _, acceptedOutput := range acceptedOutputs {
-		nextStore := store.withValue(state.slot, state.toStore + acceptedOutput)
+		nextStore := store.withValue(state.slot, append(state.toStore, acceptedOutput...))
 		outputs = append(outputs, state.next.accepting(nextStore)...)
 	}
 	return outputs
 }
 
+// Don't read in anything, just output the series of data and slots specified
 type SubexOutputState struct {
 	content []TransducerOutput
 	next SubexState
 }
-func (state SubexOutputState) build(store Store) string {
-	var builder strings.Builder
+// Given a store, return what is outputted by an epsilon transition from this state
+func (state SubexOutputState) build(store Store) []walk.Datum {
+	var result []walk.Datum
 	for _, part := range state.content {
-		builder.WriteString(part.build(store))
+		result = append(result, part.build(store)...)
 	}
-	return builder.String()
+	return result
 }
-func (state SubexOutputState) eat(store Store, char rune) []SubexBranch {
+func (state SubexOutputState) eat(store Store, char walk.Datum) []SubexBranch {
 	content := state.build(store)
 	nextStates := state.next.eat(store, char)
 	for i := range nextStates {
-		nextStates[i].output = content + nextStates[i].output
+		nextStates[i].output = append(content, nextStates[i].output...)
 	}
 	return nextStates
 }
-func (state SubexOutputState) accepting(store Store) []string {
+func (state SubexOutputState) accepting(store Store) [][]walk.Datum {
 	content := state.build(store)
 	outputs := state.next.accepting(store)
 	for i := range outputs {
-		outputs[i] = content + outputs[i]
+		outputs[i] = append(content, outputs[i]...)
 	}
 	return outputs
 }
 
+// A final state, transitions to nothing but is accepting
 type SubexNoneState struct {}
-func (state SubexNoneState) eat(store Store, char rune) []SubexBranch {
+func (state SubexNoneState) eat(store Store, char walk.Datum) []SubexBranch {
 	return nil
 }
-func (state SubexNoneState) accepting(store Store) []string {
-	return []string{""}
+func (state SubexNoneState) accepting(store Store) [][]walk.Datum {
+	return [][]walk.Datum{nil}
 }
 
+// Read in a specific datum and output it
+// TODO rename to better reflect datum instead of rune
 type SubexCopyRuneState struct {
-	rune rune
+	rune walk.Datum
 	next SubexState
 }
-func (state SubexCopyRuneState) eat(store Store, char rune) []SubexBranch {
+func (state SubexCopyRuneState) eat(store Store, char walk.Datum) []SubexBranch {
+	// TODO can I compare Datum values with == ?
 	if char == state.rune {
 		return []SubexBranch{{
 			state: state.next,
-			output: string(char),
+			output: []walk.Datum{char},
 			store: store,
 		}}
 	}
 	return nil
 }
-func (state SubexCopyRuneState) accepting(store Store) []string {
+func (state SubexCopyRuneState) accepting(store Store) [][]walk.Datum {
 	return nil
 }
 
+// Read in any datum and output it
 type SubexCopyAnyState struct {
 	next SubexState
 }
-func (state SubexCopyAnyState) eat(store Store, char rune) []SubexBranch {
+func (state SubexCopyAnyState) eat(store Store, char walk.Datum) []SubexBranch {
 	return []SubexBranch{{
 		state: state.next,
-		output: string(char),
+		output: []walk.Datum{char},
 		store: store,
 	}}
 }
-func (state SubexCopyAnyState) accepting(store Store) []string {
+func (state SubexCopyAnyState) accepting(store Store) [][]walk.Datum {
 	return nil
 }
 
+// Read in a datum and apply a map to generate a datum to output
+// If the input isn't in the map transition to nothing
 type SubexRangeState struct {
-	parts map[rune]rune
+	parts map[walk.Datum]walk.Datum
 	next SubexState
 }
-func (state SubexRangeState) eat(store Store, char rune) []SubexBranch {
+func (state SubexRangeState) eat(store Store, char walk.Datum) []SubexBranch {
 	out, exists := state.parts[char]
 	if !exists {
 		return nil
 	} else {
 		return []SubexBranch{{
 			state: state.next,
-			output: string(out),
+			output: []walk.Datum{out},
 			store: store,
 		}}
 	}
 }
-func (state SubexRangeState) accepting(store Store) []string {
+func (state SubexRangeState) accepting(store Store) [][]walk.Datum {
 	return nil
 }