@verb #280:"html2token html2token_suspended" this none this rxdo #789
@program #280:"html2token html2token_suspended" this none this
"Copied from HTML utilities (#52):html2token by Dax (#789)Jan  4 12:11:11 2000"
"Converts HTML text into a stream of tokens."
":html2token({'<OL><LI>Come <B>here</B>', '</OL>'})"
" => {'<OL>', '<LI>', 'Come ', '<B>', 'here', '</B>', '! ', '</OL>'}"
{html} = args
susp = verb == "html2token_suspended"
"Form a single long line of html"
if (typeof(html) == $LIST)
  html = $string_utils:from_list(html, " ")
endif
html = $html_utils:decomment(html)
"Break the html up into tokens"
tokens = {}
while (x = index(html, "<"))
  x > 1 && (tokens = {@tokens, html[1..x - 1]})
  if (!(y = index(html = html[x..$], ">")))
    "This tag doesn't end!"
    break
  endif
  tag = html[2..y - 1]
  "  if (!(((name = tag[1..(index(tag, \" \") || ($ + 1)) - 1]) in this.valid_tags) || ((`name[1] ! E_RANGE' == \"/\") && (name[2..$] in this.valid_tags))))"
  "    try"
  "      tokens[$] = tostr(tokens[$], strsub(strsub(html[1..y], \">\", \"&gt;\"), \"<\", \"&lt;\"));"
  "    except (E_RANGE)"
  "      tokens = {strsub(strsub(html[1..y], \">\", \"&gt;\"), \"<\", \"&lt;\")};"
  "    endtry"
  "  else"
  tokens = {@tokens, html[1..y]}
  "  endif"
  html = html[y + 1..$]
  susp && $cmd_utils:suspend_if_needed(0)
endwhile
return {@tokens, html}
"Last modified by Dax (#789) on Tue May  3 13:45:03 2005 MDT."
.