Huge refactor to a more value based system, doing away with terminals. Also introduces unit testing

1 files changed, 186 insertions, 110 deletions

diff --git a/subex/subexast.go b/subex/subexast.go
index f4088fe..31c77ba 100644
--- a/subex/subexast.go
+++ b/subex/subexast.go
@@ -7,15 +7,15 @@ import (
 
 // A node in the AST of a subex
 type SubexAST interface {
-	compileWith(next SubexState, slotMap *SlotMap) SubexState
+	compileWith(next SubexState, slotMap *SlotMap, runic bool) SubexState
 }
 
 // Process the first subex, then the second, splitting the input text in two
 type SubexASTConcat struct {
 	First, Second SubexAST
 }
-func (ast SubexASTConcat) compileWith(next SubexState, slotMap *SlotMap) SubexState {
-	return ast.First.compileWith(ast.Second.compileWith(next, slotMap), slotMap)
+func (ast SubexASTConcat) compileWith(next SubexState, slotMap *SlotMap, runic bool) SubexState {
+	return ast.First.compileWith(ast.Second.compileWith(next, slotMap, runic), slotMap, runic)
 }
 func (ast SubexASTConcat) String() string {
 	return fmt.Sprintf("(%v)(%v)", ast.First, ast.Second)
@@ -26,13 +26,21 @@ type SubexASTStore struct {
 	Match SubexAST
 	Slot rune
 }
-func (ast SubexASTStore) compileWith(next SubexState, slotMap *SlotMap) SubexState {
+func (ast SubexASTStore) compileWith(next SubexState, slotMap *SlotMap, runic bool) SubexState {
 	id := slotMap.getId(ast.Slot)
-	return &SubexCaptureBeginState {
-		next: ast.Match.compileWith(&SubexStoreEndState {
-			slot: id,
-			next: next,
-		}, slotMap),
+	newNext := ast.Match.compileWith(&SubexStoreEndState {
+		slot: id,
+		next: next,
+	}, slotMap, runic)
+
+	if !runic {
+		return &SubexCaptureBeginState {
+			next: newNext,
+		}
+	} else {
+		return &SubexCaptureRunesBeginState {
+			next: newNext,
+		}
 	}
 }
 func (ast SubexASTStore) String() string {
@@ -43,10 +51,10 @@ func (ast SubexASTStore) String() string {
 type SubexASTOr struct {
 	First, Second SubexAST
 }
-func (ast SubexASTOr) compileWith(next SubexState, slotMap *SlotMap) SubexState {
+func (ast SubexASTOr) compileWith(next SubexState, slotMap *SlotMap, runic bool) SubexState {
 	return &SubexGroupState {
-		ast.First.compileWith(next, slotMap),
-		ast.Second.compileWith(next, slotMap),
+		ast.First.compileWith(next, slotMap, runic),
+		ast.Second.compileWith(next, slotMap, runic),
 	}
 }
 func (ast SubexASTOr) String() string {
@@ -76,19 +84,19 @@ func (cr ConvexRange) decrement() ConvexRange {
 		return ConvexRange{cr.Start - 1, cr.End - 1}
 	}
 }
-func (cr ConvexRange) compile(content SubexAST, next SubexState, slotMap *SlotMap) SubexState {
+func (cr ConvexRange) compile(content SubexAST, next SubexState, slotMap *SlotMap, runic bool) SubexState {
 	min, _ := cr.minmax()
 	if min != 0 {
-		return content.compileWith(cr.decrement().compile(content, next, slotMap), slotMap)
+		return content.compileWith(cr.decrement().compile(content, next, slotMap, runic), slotMap, runic)
 	}
 	if cr.Start == -1 {
 		state := &SubexGroupState {nil, next}
-		state.first = content.compileWith(state, slotMap)
+		state.first = content.compileWith(state, slotMap, runic)
 		return state
 	}
 	if cr.End == -1 {
 		state := &SubexGroupState {next, nil}
-		state.second = content.compileWith(state, slotMap)
+		state.second = content.compileWith(state, slotMap, runic)
 		return state
 	}
 
@@ -96,7 +104,7 @@ func (cr ConvexRange) compile(content SubexAST, next SubexState, slotMap *SlotMa
 		state := next;
 		for i := 0; i < cr.Start; i += 1 {
 			state = &SubexGroupState {
-				content.compileWith(state, slotMap),
+				content.compileWith(state, slotMap, runic),
 				next,
 			}
 		}
@@ -106,7 +114,7 @@ func (cr ConvexRange) compile(content SubexAST, next SubexState, slotMap *SlotMa
 		for i := 0; i < cr.End; i += 1 {
 			state = &SubexGroupState {
 				next,
-				content.compileWith(state, slotMap),
+				content.compileWith(state, slotMap, runic),
 			}
 		}
 		return state
@@ -119,10 +127,10 @@ type SubexASTRepeat struct {
 	Content SubexAST
 	Acceptable []ConvexRange
 }
-func (ast SubexASTRepeat) compileWith(next SubexState, slotMap *SlotMap) SubexState {
+func (ast SubexASTRepeat) compileWith(next SubexState, slotMap *SlotMap, runic bool) SubexState {
 	var state SubexState = &SubexDeadState{}
 	for _, convex := range ast.Acceptable {
-		state = &SubexGroupState {state, convex.compile(ast.Content, next, slotMap)}
+		state = &SubexGroupState {state, convex.compile(ast.Content, next, slotMap, runic)}
 	}
 	return state
 }
@@ -131,23 +139,44 @@ func (ast SubexASTRepeat) String() string {
 }
 
 // Read in a single specific Atom and output it unchanged
-type SubexASTCopyAtom struct {
-	Atom walk.Atom
+type SubexASTCopyScalar struct {
+	Scalar walk.Scalar
 }
-func (ast SubexASTCopyAtom) compileWith(next SubexState, slotMap *SlotMap) SubexState {
-	return &SubexCopyAtomState{
-		atom: ast.Atom,
+func (ast SubexASTCopyScalar) compileWith(next SubexState, slotMap *SlotMap, runic bool) SubexState {
+	return &SubexCopyState{
+		filter: selectScalarFilter {ast.Scalar},
 		next: next,
 	}
 }
-func (ast SubexASTCopyAtom) String() string {
+func (ast SubexASTCopyScalar) String() string {
 	return fmt.Sprintf("a")
 }
 
+type SubexASTCopyAnyRune struct {}
+func (ast SubexASTCopyAnyRune) compileWith(next SubexState, slotMap *SlotMap, runic bool) SubexState {
+	return &SubexCopyRuneState {
+		next: next,
+		filter: anyRuneFilter{},
+	}
+}
+
+type SubexASTCopyRune struct {
+	rune rune
+}
+func (ast SubexASTCopyRune) compileWith(next SubexState, slotMap *SlotMap, runic bool) SubexState {
+	return &SubexCopyRuneState {
+		next: next,
+		filter: selectRuneFilter {ast.rune},
+	}
+}
+
 // Read in a single atom that must be a boolean and output it unchanged
 type SubexASTCopyBool struct {}
-func (ast SubexASTCopyBool) compileWith(next SubexState, slotMap *SlotMap) SubexState {
-	return &SubexCopyBoolState {next}
+func (ast SubexASTCopyBool) compileWith(next SubexState, slotMap *SlotMap, runic bool) SubexState {
+	return &SubexCopyState {
+		next: next,
+		filter: anyBoolFilter{},
+	}
 }
 func (ast SubexASTCopyBool) String() string {
 	return "?"
@@ -155,65 +184,50 @@ func (ast SubexASTCopyBool) String() string {
 
 // Read in a single atom that must be a number and output it unchanged
 type SubexASTCopyNumber struct {}
-func (ast SubexASTCopyNumber) compileWith(next SubexState, slotMap *SlotMap) SubexState {
-	return &SubexCopyNumberState {next}
+func (ast SubexASTCopyNumber) compileWith(next SubexState, slotMap *SlotMap, runic bool) SubexState {
+	return &SubexCopyState {
+		next: next,
+		filter: anyNumberFilter{},
+	}
 }
 func (ast SubexASTCopyNumber) String() string {
 	return "%"
 }
 
-// Read in a single atom that must be a string atom and output it unchanged
-type SubexASTCopyStringAtom struct {}
-func (ast SubexASTCopyStringAtom) compileWith(next SubexState, slotMap *SlotMap) SubexState {
-	return &SubexCopyStringAtomState {next}
-}
-func (ast SubexASTCopyStringAtom) String() string {
-	return "_"
-}
-
 // Read in a full string value and copy it out unchanged
 // # is equivalent to "_{-0}"
-type SubexASTCopyString struct {}
-func (ast SubexASTCopyString) compileWith(next SubexState, slotMap *SlotMap) SubexState {
-	stringAtomState := &SubexCopyStringAtomState {
-		next: nil,
-	}
-	stringContentState := &SubexGroupState {
-		&SubexCopyAtomState {
-			atom: walk.NewAtomStringTerminal(),
-			next: next,
-		},
-		stringAtomState,
-	}
-	stringAtomState.next = stringContentState
-	return &SubexCopyAtomState {
-		atom: walk.NewAtomStringTerminal(),
-		next: stringContentState,
-	}
-}
-func (ast SubexASTCopyString) String() string {
-	return "#"
-}
-
-// Read in a non-terminal value and copy it out unchanged
-// , is equivalent to `null`|?|%|#
-type SubexASTCopyValue struct {}
-func (ast SubexASTCopyValue) compileWith(next SubexState, slotMap *SlotMap) SubexState {
-	return &SubexGroupState {
-		SubexASTCopyString{}.compileWith(next, slotMap),
-		&SubexCopyNonStringNonTerminalAtomState {next},
-	}
-}
-func (ast SubexASTCopyValue) String() string {
-	return ","
-}
+// TODO
+// type SubexASTCopyString struct {}
+// func (ast SubexASTCopyString) compileWith(next SubexState, slotMap *SlotMap) SubexState {
+// 	stringAtomState := &SubexCopyStringAtomState {
+// 		next: nil,
+// 	}
+// 	stringContentState := &SubexGroupState {
+// 		&SubexCopyScalarState {
+// 			scalar: walk.NewAtomStringTerminal(),
+// 			next: next,
+// 		},
+// 		stringAtomState,
+// 	}
+// 	stringAtomState.next = stringContentState
+// 	return &SubexCopyScalarState {
+// 		scalar: walk.NewAtomStringTerminal(),
+// 		next: stringContentState,
+// 	}
+// }
+// func (ast SubexASTCopyString) String() string {
+// 	return "#"
+// }
 
 // Read in any single Atom and output it unchanged
-type SubexASTCopyAny struct {}
-func (ast SubexASTCopyAny) compileWith(next SubexState, slotMap *SlotMap) SubexState {
-	return &SubexCopyAnyState{next}
+type SubexASTCopyAnyValue struct {}
+func (ast SubexASTCopyAnyValue) compileWith(next SubexState, slotMap *SlotMap, runic bool) SubexState {
+	return &SubexCopyState {
+		next: next,
+		filter: anyValueFilter{},
+	}
 }
-func (ast SubexASTCopyAny) String() string {
+func (ast SubexASTCopyAnyValue) String() string {
 	return "."
 }
 
@@ -228,10 +242,10 @@ func (ast OutputLoadAST) compile(slotMap *SlotMap) OutputContent {
 	return OutputLoad {slotMap.getId(ast.slot)}
 }
 
-type OutputAtomLiteralAST struct {
-	atom walk.Atom
+type OutputValueLiteralAST struct {
+	atom walk.Value
 }
-func (ast OutputAtomLiteralAST) compile(slotMap *SlotMap) OutputContent {
+func (ast OutputValueLiteralAST) compile(slotMap *SlotMap) OutputContent {
 	return OutputAtomLiteral {ast.atom}
 }
 
@@ -239,7 +253,7 @@ func (ast OutputAtomLiteralAST) compile(slotMap *SlotMap) OutputContent {
 type SubexASTOutput struct {
 	Replacement []OutputContentAST
 }
-func (ast SubexASTOutput) compileWith(next SubexState, slotMap *SlotMap) SubexState {
+func (ast SubexASTOutput) compileWith(next SubexState, slotMap *SlotMap, runic bool) SubexState {
 	var content []OutputContent
 	for _, el := range ast.Replacement {
 		content = append(content, el.compile(slotMap))
@@ -257,13 +271,13 @@ func (ast SubexASTOutput) String() string {
 type SubexASTJoin struct {
 	Content, Delimiter SubexAST
 }
-func (ast SubexASTJoin) compileWith(next SubexState, slotMap *SlotMap) SubexState {
+func (ast SubexASTJoin) compileWith(next SubexState, slotMap *SlotMap, runic bool) SubexState {
 	afterContentState := &SubexGroupState {
 		nil,
 		next,
 	}
-	manyContentsState := ast.Content.compileWith(afterContentState, slotMap)
-	afterContentState.first = ast.Delimiter.compileWith(manyContentsState, slotMap)
+	manyContentsState := ast.Content.compileWith(afterContentState, slotMap, runic)
+	afterContentState.first = ast.Delimiter.compileWith(manyContentsState, slotMap, runic)
 	return &SubexGroupState {
 		manyContentsState,
 		next,
@@ -275,30 +289,30 @@ func (ast SubexASTJoin) String() string {
 
 // Run each input Atom through a map to produce an output Atom
 // Atoms not in the map cause this to not match
-type SubexASTRange struct {
-	Parts map[walk.Atom]walk.Atom
-}
-func (ast SubexASTRange) compileWith(next SubexState, slotMap *SlotMap) SubexState {
-	return &SubexRangeState {
-		parts: ast.Parts,
-		next: next,
-	}
-}
-func (ast SubexASTRange) String() string {
-	return fmt.Sprintf("[abc=xyz]")
-}
+// type SubexASTRange struct {
+// 	Parts map[walk.Atom]walk.Atom
+// }
+// func (ast SubexASTRange) compileWith(next SubexState, slotMap *SlotMap) SubexState {
+// 	return &SubexRangeState {
+// 		parts: ast.Parts,
+// 		next: next,
+// 	}
+// }
+// func (ast SubexASTRange) String() string {
+// 	return fmt.Sprintf("[abc=xyz]")
+// }
 
 // Run content, if content is a list of booleans, OR them, if all values are castable to numbers, sum them and output the total
 // Reject if neither of these cases match
 type SubexASTSum struct {
 	Content SubexAST
 }
-func (ast SubexASTSum) compileWith(next SubexState, slotMap *SlotMap) SubexState {
+func (ast SubexASTSum) compileWith(next SubexState, slotMap *SlotMap, runic bool) SubexState {
 	return &SubexCaptureBeginState {
 		next: ast.Content.compileWith(&SubexArithmeticEndState {
 			next: next,
 			calculate: sumValues,
-		}, slotMap),
+		}, slotMap, runic),
 	}
 }
 func (ast SubexASTSum) String() string {
@@ -309,12 +323,12 @@ func (ast SubexASTSum) String() string {
 type SubexASTProduct struct {
 	Content SubexAST
 }
-func (ast SubexASTProduct) compileWith(next SubexState, slotMap *SlotMap) SubexState {
+func (ast SubexASTProduct) compileWith(next SubexState, slotMap *SlotMap, runic bool) SubexState {
 	return &SubexCaptureBeginState {
 		next: ast.Content.compileWith(&SubexArithmeticEndState {
 			next: next,
 			calculate: multiplyValues,
-		}, slotMap),
+		}, slotMap, runic),
 	}
 }
 func (ast SubexASTProduct) String() string {
@@ -326,12 +340,12 @@ func (ast SubexASTProduct) String() string {
 type SubexASTNegate struct {
 	Content SubexAST
 }
-func (ast SubexASTNegate) compileWith(next SubexState, slotMap *SlotMap) SubexState {
+func (ast SubexASTNegate) compileWith(next SubexState, slotMap *SlotMap, runic bool) SubexState {
 	return &SubexCaptureBeginState {
 		next: ast.Content.compileWith(&SubexArithmeticEndState {
 			next: next,
 			calculate: negateValues,
-		}, slotMap),
+		}, slotMap, runic),
 	}
 }
 func (ast SubexASTNegate) String() string {
@@ -344,12 +358,12 @@ func (ast SubexASTNegate) String() string {
 type SubexASTReciprocal struct {
 	Content SubexAST
 }
-func (ast SubexASTReciprocal) compileWith(next SubexState, slotMap *SlotMap) SubexState {
+func (ast SubexASTReciprocal) compileWith(next SubexState, slotMap *SlotMap, runic bool) SubexState {
 	return &SubexCaptureBeginState {
 		next: ast.Content.compileWith(&SubexArithmeticEndState {
 			next: next,
 			calculate: reciprocalValues,
-		}, slotMap),
+		}, slotMap, runic),
 	}
 }
 func (ast SubexASTReciprocal) String() string {
@@ -362,12 +376,12 @@ func (ast SubexASTReciprocal) String() string {
 type SubexASTNot struct {
 	Content SubexAST
 }
-func (ast SubexASTNot) compileWith(next SubexState, slotMap *SlotMap) SubexState {
+func (ast SubexASTNot) compileWith(next SubexState, slotMap *SlotMap, runic bool) SubexState {
 	return &SubexCaptureBeginState {
 		next: ast.Content.compileWith(&SubexArithmeticEndState {
 			next: next,
 			calculate: notValues,
-		}, slotMap),
+		}, slotMap, runic),
 	}
 }
 func (ast SubexASTNot) String() string {
@@ -376,7 +390,7 @@ func (ast SubexASTNot) String() string {
 
 // Does nothing
 type SubexASTEmpty struct {}
-func (ast SubexASTEmpty) compileWith(next SubexState, slotMap *SlotMap) SubexState {
+func (ast SubexASTEmpty) compileWith(next SubexState, slotMap *SlotMap, runic bool) SubexState {
 	return next
 }
 func (ast SubexASTEmpty) String() string {
@@ -387,11 +401,73 @@ func (ast SubexASTEmpty) String() string {
 type SubexASTDiscard struct {
 	Content SubexAST
 }
-func (ast SubexASTDiscard) compileWith(next SubexState, slotMap *SlotMap) SubexState {
-	return &SubexCaptureBeginState {
-		next: ast.Content.compileWith(&SubexDiscardState {next}, slotMap),
+func (ast SubexASTDiscard) compileWith(next SubexState, slotMap *SlotMap, runic bool) SubexState {
+	newNext := ast.Content.compileWith(&SubexDiscardState {next}, slotMap, runic)
+	if !runic {
+		return &SubexCaptureBeginState {
+			next: newNext,
+		}
+	} else {
+		return &SubexCaptureRunesBeginState {
+			next: newNext,
+		}
 	}
 }
 func (ast SubexASTDiscard) String() string {
 	return fmt.Sprintf("(%v)$_", ast.Content)
 }
+
+// Go into an array, pass the content each of the values in the array to eat and then leave the array
+type SubexASTEnterArray struct {
+	Content SubexAST
+}
+func (ast SubexASTEnterArray) compileWith(next SubexState, slotMap *SlotMap, runic bool) SubexState {
+	return &SubexCaptureBeginState {
+		next: &SubexIncrementNestState {
+			next: &SubexCopyState {
+				filter: anyArrayFilter{},
+				next: &SubexDiscardState {
+					next: &SubexCaptureBeginState {
+						next: ast.Content.compileWith(
+							&SubexDiscardTerminalState {
+								terminal: walk.ArrayEndTerminal{},
+								next: &SubexDecrementNestState {
+									next: &SubexConstructArrayState {next: next},
+								},
+							},
+							slotMap,
+							runic,
+						),
+					},
+				},
+			},
+		},
+	}
+}
+
+type SubexASTEnterString struct {
+	Content SubexAST
+}
+func (ast SubexASTEnterString) compileWith(next SubexState, slotMap *SlotMap, runic bool) SubexState {
+	return &SubexCaptureBeginState {
+		next: &SubexIncrementNestState {
+			next: &SubexCopyState {
+				filter: anyStringFilter{},
+				next: &SubexDiscardState {
+					next: &SubexCaptureRunesBeginState {
+						next: ast.Content.compileWith(
+							&SubexDecrementNestState {
+								next: &SubexDiscardTerminalState {
+									terminal: walk.StringEndTerminal{},
+									next: &SubexConstructStringState {next: next},
+								},
+							},
+							slotMap,
+							true,
+						),
+					},
+				},
+			},
+		},
+	}
+}