00100 rem ************************************************************
00110 rem *** tokens.bas - Tokenizer subroutine demo
00120 rem *** 2016 David Meyer <papa@sdf.org>  +JMJ
00130 rem ************************************************************
00140 dim t1$(20)
00150 print "Enter token string";
00160 input t0$
00170 gosub 230
00180 print "Token count:", t1
00190 for i = 1 to t1
00200 print i; ":", t1$(i)
00210 next i
00220 stop
00230 rem ************************************************************
00240 rem * Tokenize subroutine
00250 rem * Split character string into up to 20 one-or-more-space-
00260 rem * separated substrings (tokens)
00270 rem * IN:
00280 rem *  t0$ - Token string
00290 rem * OUT:
00300 rem *  t1 - Token count
00310 rem *  t1$() - Token array
00320 rem * LOCAL:
00330 rem *  c$ - Current character from t0$
00340 rem *  d - Token detected flag
00350 rem ************************************************************
00360 t1 = 0
00370 d = 0
00380 rem * For each character in t0$ ...
00390 for i = 1 to len(t0$)
00400 c$ = mid$(t0$,i,1)
00410 rem * If current character not space ...
00420 if c$ = " " then goto 570
00430 rem * If token not currently detected (beginning or between tokens) ...
00440 if d then goto 520
00450 rem * Set token detected flag
00460 d = 1
00470 rem * Increment token count
00480 t1 = t1 + 1
00490 rem * Initialize new token with current character
00500 t1$(t1) = c$
00510 goto 550
00520 rem ELSE (token currently detected) ...
00530 rem * Append current character to current token
00540 t1$(t1) = t1$(t1) + c$
00550 rem END IF
00560 goto 600
00570 rem ELSE (current character is space) ...
00580 rem * Unset token detected flag
00590 d = 0
00600 rem END IF
00610 next i
00620 return
00630 end