I think I got the tokenizer right.

implementations/Ocaml/helloworld/bin/main.ml

@@ -19,8 +19,76 @@ let rec joy_to_string jt =
   | JoyTrue -> "true"
   | JoyFalse -> "false"
   | JoyInt i -> string_of_int i
-  | JoyList el -> "[" ^ expression_to_joy el ^ "]"
+  | JoyList el -> "[" ^ expression_to_string el ^ "]"
 
-and expression_to_joy el = String.concat " " (List.map joy_to_string el)
+and expression_to_string el = String.concat " " (List.map joy_to_string el)
 
-let () = print_endline (joy_to_string dummy)
+
+
+type token =
+  | Left_bracket
+  | Right_bracket
+  | Token of string
+
+let delimiter ch = String.contains "[] " ch
+
+(* string -> int -> int -> token * int *)
+let rec tokenize1 str start last =
+  if last >= String.length str || delimiter (String.get str last)
+  then (Token (String.sub str start (last - start)), last)
+  else tokenize1 str start (last + 1)
+
+let rec tokenize0 str start acc =
+  if start >= String.length str
+  then acc
+  else
+    let ch = String.get str start in
+      match ch with
+      | '[' -> Left_bracket :: (tokenize0 str (start + 1) acc)
+      | ']' -> Right_bracket :: (tokenize0 str (start + 1) acc)
+      | ' ' -> tokenize0 str (start + 1) acc
+      | _ -> let (token, n) = tokenize1 str start (start + 1) in
+        token :: (tokenize0 str n acc)
+
+let tokenize str = tokenize0 str 0 []
+
+
+let token_to_string token =
+  match token with
+  | Left_bracket -> "["
+  | Right_bracket -> "]"
+  | Token str -> str
+
+
+(*
+let char_tok ch acc =
+  match ch with
+  | '[' -> Left_bracket :: acc
+  | ']' -> Right_bracket :: acc
+  | ' ' -> acc
+  | x -> (Token x) :: acc
+
+let tokenize str = 
+  String.fold_right char_tok str []
+
+let text_to_expression str =
+  let tokens = tokenize str in
+    tokens
+
+let token_to_string token = 
+  match token with
+  | Left_bracket -> "["
+  | Right_bracket -> "]"
+  | Token x -> Char.escaped x
+    
+let s = String.concat "" (List.map token_to_string (text_to_expression "1 [2]3" ))
+*)
+
+(* let () = print_endline (joy_to_string dummy) *)
+
+let s = String.concat " " (List.map token_to_string (tokenize "1 [2]3" ))
+
+
+let () = 
+  print_endline s ;
+  print_endline (joy_to_string dummy)