From 6fc77a9a4a8dac863025d6ed8b5666adfb3250b8 Mon Sep 17 00:00:00 2001
From: Simon Forman <sforman@hushmail.com>
Date: Tue, 6 Apr 2021 12:40:04 -0700
Subject: [PATCH] Simplify tokenizing, raise Unknown errors.

This brings the behaviour of Python Joy into congruence with Nim Joy.
---
 joy/joy.py    | 10 +++++++++-
 joy/parser.py | 39 ++++++++++++++++++---------------------
 2 files changed, 27 insertions(+), 22 deletions(-)

diff --git a/joy/joy.py b/joy/joy.py
index 4ef38e8..88a9cbf 100644
--- a/joy/joy.py
+++ b/joy/joy.py
@@ -29,6 +29,9 @@ from .parser import text_to_expression, ParseError, Symbol
 from .utils.stack import stack_to_string
 
 
+class UnknownSymbolError(KeyError): pass
+
+
 def joy(stack, expression, dictionary, viewer=None):
 	'''Evaluate a Joy expression on a stack.
 
@@ -53,7 +56,10 @@ def joy(stack, expression, dictionary, viewer=None):
 
 		term, expression = expression
 		if isinstance(term, Symbol):
-			term = dictionary[term]
+			try:
+				term = dictionary[term]
+			except KeyError:
+				raise UnknownSymbolError(term)
 			stack, expression, dictionary = term(stack, expression, dictionary)
 		else:
 			stack = term, stack
@@ -123,6 +129,8 @@ def interp(stack=(), dictionary=None):
 				break
 			try:
 				stack, _, dictionary = run(text, stack, dictionary)
+			except UnknownSymbolError as sym:
+				print('Unknown:', sym)
 			except:
 				print_exc()
 			print(stack_to_string(stack))
diff --git a/joy/parser.py b/joy/parser.py
index ccd68dc..e13eb9d 100644
--- a/joy/parser.py
+++ b/joy/parser.py
@@ -40,15 +40,16 @@ from re import Scanner
 from .utils.stack import list_to_stack
 
 
-#TODO: explain the details of float lits and strings.
-BOOL = 'true|false'
-FLOAT = r'-?\d+\.\d*(e(-|\+)\d+)?'
-INT = r'-?\d+'
-SYMBOL = r'[•\w!@$%^&*()_+<>?|\/;:`~,.=-]+'
 BRACKETS = r'\[|\]'
-STRING_DOUBLE_QUOTED = r'"(?:[^"\\]|\\.)*"'
-STRING_SINGLE_QUOTED = r"'(?:[^'\\]|\\.)*'"
 BLANKS = r'\s+'
+WORDS = r'[^[\]\s]+'
+
+
+token_scanner = Scanner([
+    (BRACKETS, lambda _, token: token),
+    (BLANKS, None),
+    (WORDS, lambda _, token: token),
+    ])
 
 
 class Symbol(str):
@@ -81,7 +82,7 @@ def _tokenize(text):
 
 	Raise ParseError (with some of the failing text) if the scan fails.
 	'''
-	tokens, rest = _scanner.scan(text)
+	tokens, rest = token_scanner.scan(text)
 	if rest:
 		raise ParseError(
 			'Scan failed at position %i, %r'
@@ -107,20 +108,16 @@ def _parse(tokens):
 			except IndexError:
 				raise ParseError('Extra closing bracket.')
 			frame[-1] = list_to_stack(frame[-1])
+		elif tok == 'true':
+			frame.append(True)
+		elif tok == 'false':
+			frame.append(False)
 		else:
-			frame.append(tok)
+			try:
+				thing = int(tok)
+			except ValueError:
+				thing = Symbol(tok)
+			frame.append(thing)
 	if stack:
 		raise ParseError('Unclosed bracket.')
 	return list_to_stack(frame)
-
-
-_scanner = Scanner([
-	(                BOOL, lambda _, token: token == 'true'),
-	(               FLOAT, lambda _, token: float(token)),
-	(                 INT, lambda _, token: int(token)),
-	(              SYMBOL, lambda _, token: Symbol(token)),
-	(            BRACKETS, lambda _, token: token),
-	(STRING_DOUBLE_QUOTED, lambda _, token: token[1:-1].replace('\\"', '"')),
-	(STRING_SINGLE_QUOTED, lambda _, token: token[1:-1].replace("\\'", "'")),
-	(              BLANKS, None),
-	])