On the feature/redrawhook branch, advise its users that they can switch back to master.

Fixes #655.

.editorconfig

@@ -7,3 +7,9 @@ end_of_line = lf
 tab_width   = 2
 indent_size = 2
 indent_style = space
+
+[Makefile]
+tab_width   = 8
+indent_size = 8
+indent_style = tab
+

.travis.yml

@@ -3,6 +3,7 @@ sudo: required
 
 env:
   - ZSH=master
+  - ZSH=5.8
   - ZSH=5.7.1
   - ZSH=5.7
   - ZSH=5.6.2
@@ -35,7 +36,7 @@ env:
   - ZSH=4.3.12
   - ZSH=4.3.11
 
-script: docker run -v $PWD:/work -w /work zshusers/zsh:${ZSH} /bin/sh -c 'install_packages make procps && make test'
+script: docker run -v $PWD:/work -w /work zshusers/zsh:${ZSH} /bin/sh -c 'install_packages make procps bsdmainutils && make test'
 
 notifications:
   webhooks:

.version

@@ -1 +1 @@
-0.7.0
+0.8.0-alpha2-dev

HACKING.md

@@ -67,6 +67,22 @@ expected_region_highlight=(
 )
 ```
 
+Memos and commas
+----------------
+
+We append to `region_highlight` as follows:
+
+
+```zsh
+region_highlight+=("$start $end $spec, memo=zsh-syntax-highlighting")
+```
+
+That comma is required to cause zsh 5.8 and older to ignore the memo without
+ignoring the `$spec`.  It's a hack, but given that no further 5.8.x patch
+releases are planned, it's been deemed acceptable.  See issue #418 and the
+cross-referenced issues.
+
+
 Miscellany
 ----------
 

INSTALL.md

@@ -23,7 +23,7 @@ How to install
 [ubuntu-package]: https://launchpad.net/ubuntu/+source/zsh-syntax-highlighting
 [fedora-package]: https://apps.fedoraproject.org/packages/zsh-syntax-highlighting
 [fedora-package-alt]: https://bodhi.fedoraproject.org/updates/?packages=zsh-syntax-highlighting
-[obs-repository]: https://software.opensuse.org//download.html?project=shells%3Azsh-users%3Azsh-syntax-highlighting&package=zsh-syntax-highlighting
+[obs-repository]: https://software.opensuse.org/download.html?project=shells%3Azsh-users%3Azsh-syntax-highlighting&package=zsh-syntax-highlighting
 [void-package]: https://github.com/void-linux/void-packages/tree/master/srcpkgs/zsh-syntax-highlighting
 
 See also [repology's cross-distro index](https://repology.org/metapackage/zsh-syntax-highlighting/versions)

Makefile

@@ -41,7 +41,7 @@ test:
 	for test in highlighters/*; do \
 		if [ -d $$test/test-data ]; then \
 			echo "Running test $${test##*/}"; \
-			$(ZSH) -f tests/test-highlighting.zsh "$${test##*/}"; \
+			env -i QUIET=$$QUIET $${TERM:+"TERM=$$TERM"} $(ZSH) -f tests/test-highlighting.zsh "$${test##*/}"; \
 			: $$(( result |= $$? )); \
 		fi \
 	done; \

README.md

@@ -27,6 +27,11 @@ Before: [![Screenshot #3.1](images/before3-smaller.png)](images/before3.png)
 <br/>
 After:&nbsp; [![Screenshot #3.2](images/after3-smaller.png)](images/after3.png)
 
+Before: [![Screenshot #4.1](images/before4-smaller.png)](images/before4-smaller.png)
+<br/>
+After:&nbsp; [![Screenshot #4.2](images/after4-smaller.png)](images/after4-smaller.png)
+
+
 
 How to install
 --------------
@@ -39,11 +44,23 @@ FAQ
 
 ### Why must `zsh-syntax-highlighting.zsh` be sourced at the end of the `.zshrc` file?
 
-`zsh-syntax-highlighting.zsh` wraps ZLE widgets.  It must be sourced after all
+zsh-syntax-highlighting works by hooking into the Zsh Line Editor (ZLE) and
-custom widgets have been created (i.e., after all `zle -N` calls and after
+computing syntax highlighting for the command-line buffer as it stands at the
-running `compinit`).  Widgets created later will work, but will not update the
+time z-sy-h's hook is invoked.
+
+In zsh 5.2 and older,
+`zsh-syntax-highlighting.zsh` hooks into ZLE by wrapping ZLE widgets.  It must
+be sourced after all custom widgets have been created (i.e., after all `zle -N`
+calls and after running `compinit`) in order to be able to wrap all of them.
+Widgets created after z-sy-h is sourced will work, but will not update the
 syntax highlighting.
 
+In zsh newer than 5.8 (not including 5.8 itself),
+zsh-syntax-highlighting uses the `add-zle-hook-widget` facility to install
+a `zle-line-pre-redraw` hook.  Hooks are run in order of registration,
+therefore, z-sy-h must be sourced (and register its hook) after anything else
+that adds hooks that modify the command-line buffer.
+
 ### Does syntax highlighting work during incremental history search?
 
 Highlighting the command line during an incremental history search (by default bound to

changelog.md

@@ -1,6 +1,199 @@
-# Changes in version 0.7.0
+# Changes in HEAD
 
-**Version 0.7.0 has not been released.  This changelog is a work in progress.**
+
+## Changes fixed as part of the switch to zle-line-pre-redraw
+
+The changes in this section were fixed by switching to a `zle-line-pre-redraw`-based
+implementation.
+
+Note: The new implementation will only be used on future zsh releases,
+numbered 5.8.0.3 and newer, due to interoperability issues with other plugins
+(issues #418 and #579).  The underlying zsh feature has been available since
+zsh 5.2.
+
+Whilst under development, the new implementation was known as the
+"feature/redrawhook" topic branch.
+
+- Fixed: Highlighting not triggered after popping a buffer from the buffer stack
+  (using the `push-line` widget, default binding: `M-q`)
+  [#40]
+
+- Fixed: Invoking completion when there were no matches removed highlighting
+  [#90, #470]
+
+- Fixed: Two successive deletes followed by a yank only yanked the latest
+  delete, rather than both of them
+  [#150, #151, #160; cf. #183]
+
+- Presumed fixed: Completing `$(xsel)` results in an error message from `xsel`,
+  with pre-2017 versions of `xsel`.  (For 2017 vintage and newer, see the issue
+  for details.)
+  [#154]
+
+- Fixed: When the standard `bracketed-paste-magic` widget is in use, pastes were slow
+  [#295]
+
+- Fixed: No way to prevent a widget from being wrapped
+  [#324]
+
+- Fixed: No highlighting while cycling menu completion
+  [#375]
+
+- Fixed: Does not coexist with the `IGNORE_EOF` option
+  [#377]
+
+- Fixed: The `undefined-key` widget was wrapped
+  [#421]
+
+- Fixed: Does not coexist with the standard `surround` family of widgets
+  [#520]
+
+- Fixed: First completed filename doesn't get `path` highlighting
+  [#632]
+
+
+# Changes in 0.8.0-alpha1-pre-redrawhook
+
+## Notice about an improbable-but-not-impossible forward incompatibility
+
+Everyone can probably skip this section.
+
+The `master` branch of zsh-syntax-highlighting uses a zsh feature that has not
+yet appeared in a zsh release: the `memo=` feature, added to zsh in commit
+zsh-5.8-172-gdd6e702ee (after zsh 5.8, before zsh 5.9).  In the unlikely event
+that this zsh feature should change in an incompatible way before the next
+stable zsh release, set `zsh_highlight__memo_feature=0` in your .zshrc files to
+disable use of the new feature.
+
+z-sy-h dogfoods the new, unreleased zsh feature because that feature was
+added to zsh at z-sy-h's initiative.  The new feature is used in the fix
+to issue #418.
+
+
+## Incompatible changes:
+
+- An unsuccessful completion (a <kbd>⮀ Tab</kbd> press that doesn't change the
+  command line) no longer causes highlighting to be lost.  Visual feedback can
+  alternatively be achieved by setting the `format` zstyle under the `warnings`
+  tag, for example,
+
+        zstyle ':completion:*:warnings' format '%F{red}No matches%f'
+
+    Refer to the [description of the `format` style in `zshcompsys(1)`]
+    [zshcompsys-Standard-Styles-format].
+
+    (#90, part of #245 (feature/redrawhook))
+
+[zshcompsys-Standard-Styles]: http://zsh.sourceforge.net/Doc/Release/Completion-System.html#Standard-Styles
+[zshcompsys-Standard-Styles-format]: http://zsh.sourceforge.net/Doc/Release/Completion-System.html#index-format_002c-completion-style
+  
+
+
+## Other changes:
+
+- Document `$ZSH_HIGHLIGHT_MAXLENGTH`.
+  [#698]
+
+- Optimize highlighting unquoted words (words that are not in single quotes, double quotes, backticks, or dollar-single-quotes)
+  [#730]
+
+- Redirection operators (e.g., `<` and `>`) are now highlighted by default
+  [#646]
+
+- Propertly terminate `noglob` scope in try/always blocks
+  [#577]
+
+- Don't error out when `KSH_ARRAYS` is set in the calling scope
+  [#622, #689]
+
+- Literal semicolons in array assignments (`foo=( bar ; baz )`) are now
+  highlighted as errors.
+  [3ca93f864fb6]
+
+- Command separators in array assignments (`foo=( bar | baz )`) are now
+  highlighted as errors.
+  [#651, 81267ca3130c]
+
+- Support parameter elision in command position (e.g., `$foo ls` where `$foo` is unset or empty)
+  [#667]
+
+- Don't consider the filename in `sudo -e /path/to/file` to be a command position
+  [#678]
+
+- Don't look up absolute directory names in $cdpath
+  [2cc2583f8f12, part of #669]
+
+- Fix `exec 2>&1;` being highlighted as an error.
+  [#676]
+
+- Fix `: $(<*)` being highlighted as globbing.
+  [#582]
+
+- Fix `cat < *` being highlighting as globbing when the `MULTIOS` option is unset.
+  [#583]
+
+- Fix `echo >&2` highlighting the `2` as a filename if a file by that name happened to exist
+  [#694, part of #645]
+
+- Fix `echo >&-` highlighting the `-` as a filename if a file by that name happened to exist
+  [part of #645]
+
+- Fix `echo >&p` highlighting the `p` as a filename if a file by that name happened to exist
+  [part of #645]
+
+- Fix wrong highlighting of unquoted parameter expansions under zsh 5.2 and older
+  [e165f18c758e]
+
+- Highlight global aliases
+  [#700]
+
+- Highlight `: =nosuchcommand' as an error (when the `EQUALS` option hasn't been unset).
+  [#430]
+
+- Highlight reserved word after assignments as errors (e.g., `foo=bar (ls;)`)
+  [#461]
+
+- Correctly highlight `[[ foo && bar || baz ]]`.
+
+- Highlight non-executable files in command position correctly (e.g., `% /etc/passwd`)
+  [#202, #669]
+
+- Highlight directories in command position correctly, including `AUTO_CD` support
+  [#669]
+
+- Recognize `env` as a precommand (e.g., `env FOO=bar ls`)
+
+- Recognize `strace` as a precommand
+
+- Fix an error message on stderr before every prompt when the `WARN_NESTED_VAR` zsh option is set:
+  `_zsh_highlight_main__precmd_hook:1: array parameter _zsh_highlight_main__command_type_cache set in enclosing scope in function _zsh_highlight_main__precmd_hook`
+  [#727, #731, #732, #733]
+
+- Fix highlighting of alias whose definitions use a simple command terminator
+  (such as `;`, `|`, `&&`) before a newline
+  [#677; had regressed in 0.7.0]
+
+- Highlight arithmetic expansions (e.g., `$(( 42 ))`)
+  [#607 #649 #704]
+
+- Highlight the parentheses of array assignments as reserved words (`foo=( bar )`).
+  The `assign` style remains supported and has precedence.
+  [#585]
+
+- Fix interoperability issue with other plugins that use highlighting.  The fix
+  requires zsh 5.8.0.3 or newer.  (zsh 5.8.0.2-dev from the `master` branch,
+  revision zsh-5.8-172-gdd6e702ee or newer is also fine.)
+  [#418, https://github.com/okapia/zsh-viexchange/issues/1]
+
+- Improve performance of the `brackets` highlighter.
+
+
+# Changes in version 0.7.1
+
+- Remove out-of-date information from the 0.7.0 changelog.
+
+
+# Changes in version 0.7.0
 
 This is a stable bugfix and feature release.  Major new features and changes include:
 
@@ -32,7 +225,7 @@ This is a stable bugfix and feature release.  Major new features and changes inc
 
 - Highlight numeric globs (e.g., `echo /lib<->`)
 
-- Assorted improvement to aliases highlighting
+- Assorted improvements to aliases highlighting
   (e.g.,
    `alias sudo_u='sudo -u'; sudo_u jrandom ls`,
    `alias x=y y=z z=nosuchcommand; x`,
@@ -70,13 +263,7 @@ Known issues include:
   before a newline will incorrectly be highlighted as an error.  See issue #677
   for examples and workarounds.
   [#677]
-
+  [UPDATE: Fixed in 0.8.0]
-- Use of a simple parameter expansion (`${foo}` with nothing but a parameter
-  name inside the braces) in command position, when the value of the parameter
-  looks like an assignment (such as after `foo='bar=$(ls)'`), may result in
-  incorrect highlighting and "BUG" messages.  As a workaround, the expansion
-  may be nested (`${${foo}}`).
-  [#670]
 
 
 # Changes in version 0.6.0

docs/highlighters.md

@@ -18,6 +18,23 @@ Syntax highlighting is done by pluggable highlighters:
 [6]: highlighters/line.md
 
 
+Highlighter-independent settings
+--------------------------------
+
+By default, all command lines are highlighted.  However, it is possible to
+prevent command lines longer than a fixed number of characters from being
+highlighted by setting the variable `${ZSH_HIGHLIGHT_MAXLENGTH}` to the maximum
+length (in characters) of command lines to be highlighter.  This is useful when
+editing very long comand lines (for example, with the [`fned`][fned] utility
+function).  Example:
+
+[fned]: http://zsh.sourceforge.net/Doc/Release/User-Contributions.html#index-zed
+
+```zsh
+ZSH_HIGHLIGHT_MAXLENGTH=512
+```
+
+
 How to activate highlighters
 ----------------------------
 

docs/highlighters/main.md

@@ -20,12 +20,14 @@ This highlighter defines the following styles:
 * `reserved-word` - shell reserved words (`if`, `for`)
 * `alias` - aliases
 * `suffix-alias` - suffix aliases (requires zsh 5.1.1 or newer)
+* `global-alias` - global aliases
 * `builtin` - shell builtin commands (`shift`, `pwd`, `zstyle`)
 * `function` - function names
 * `command` - command names
 * `precommand` - precommand modifiers (e.g., `noglob`, `builtin`)
 * `commandseparator` - command separation tokens (`;`, `&&`)
 * `hashed-command` - hashed commands
+* `autodirectory` - a directory name in command position when the `AUTO_CD` option is set
 * `path` - existing filenames
 * `path_pathseparator` - path separators in filenames (`/`); if unset, `path` is used (default)
 * `path_prefix` - prefixes of existing filenames
@@ -40,6 +42,7 @@ This highlighter defines the following styles:
 * `command-substitution-delimiter-quoted` - a quoted command substitution delimiters (`"$(` and `)"`)
 * `process-substitution` - process substitutions (`<(echo foo)`)
 * `process-substitution-delimiter` - process substitution delimiters (`<(` and `)`)
+* `arithmetic-expansion` - arithmetic expansion `$(( 42 ))`)
 * `single-hyphen-option` - single-hyphen options (`-o`)
 * `double-hyphen-option` - double-hyphen options (`--option`)
 * `back-quoted-argument` - backtick command substitution (`` `foo` ``)
@@ -58,7 +61,9 @@ This highlighter defines the following styles:
 * `assign` - parameter assignments (`x=foo` and `x=( )`)
 * `redirection` - redirection operators (`<`, `>`, etc)
 * `comment` - comments, when `setopt INTERACTIVE_COMMENTS` is in effect (`echo # foo`)
-* `named-fd` - named file descriptor (`echo foo {fd}>&2`)
+* `comment` - elided parameters in command position (`$x ls` when `$x` is unset or empty)
+* `named-fd` - named file descriptor (the `fd` in `echo foo {fd}>&2`)
+* `numeric-fd` - numeric file descriptor (the `2` in `echo foo {fd}>&2`)
 * `arg0` - a command word other than one of those enumerated above (other than a command, precommand, alias, function, or shell builtin command).
 * `default` - everything else
 

docs/highlighters/regexp.md

@@ -11,7 +11,7 @@ To use this highlighter, associate regular expressions with styles in the
 `ZSH_HIGHLIGHT_REGEXP` associative array, for example in `~/.zshrc`:
 
 ```zsh
-typeset -A ZSH_HIGHLIGHT_PATTERNS
+typeset -A ZSH_HIGHLIGHT_REGEXP
 ZSH_HIGHLIGHT_REGEXP+=('\bsudo\b' fg=123,bold)
 ```
 

highlighters/brackets/brackets-highlighter.zsh

@@ -51,8 +51,9 @@ _zsh_highlight_highlighter_brackets_paint()
   local -A levelpos lastoflevel matching
 
   # Find all brackets and remember which one is matching
-  for (( pos = 1; pos <= buflen; pos++ )) ; do
+  pos=0
-    char=$BUFFER[pos]
+  for char in ${(s..)BUFFER} ; do
+    (( ++pos ))
     case $char in
       ["([{"])
         levelpos[$pos]=$((++level))

highlighters/main/test-data/abspath-in-command-position1.zsh

@@ -31,5 +31,5 @@
 BUFFER=$'/'
 
 expected_region_highlight=(
-  '1 1 path' # /
+  '1 1 path_prefix' # /
 )

highlighters/main/test-data/abspath-in-command-position1b.zsh

@@ -0,0 +1,36 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+setopt autocd
+BUFFER=$'/'
+
+expected_region_highlight=(
+  '1 1 autodirectory' # /
+)

highlighters/main/test-data/abspath-in-command-position3.zsh

@@ -28,8 +28,10 @@
 # vim: ft=zsh sw=2 ts=2 et
 # -------------------------------------------------------------------------------------------------
 
-BUFFER=$'/bin'
+BUFFER=$'/bin; /bin'
 
 expected_region_highlight=(
-  '1 4 path' # /bin
+  '1 4 unknown-token' # /bin (in middle)
+  '5 5 commandseparator' # ;
+  '7 10 path_prefix' # /bin (at end)
 )

highlighters/main/test-data/abspath-in-command-position3b.zsh

@@ -0,0 +1,38 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+setopt autocd
+BUFFER=$'/bin; /bin'
+
+expected_region_highlight=(
+  '1 4 autodirectory' # /bin (in middle)
+  '5 5 commandseparator' # ;
+  '7 10 autodirectory' # /bin (at end)
+)

highlighters/main/test-data/alias-comment1.zsh

@@ -27,11 +27,11 @@
 # vim: ft=zsh sw=2 ts=2 et
 # -------------------------------------------------------------------------------------------------
 
-# see alias-comment2.zsh
+# see alias-comment2.zsh and comment-followed.zsh
 setopt interactivecomments
 alias x=$'# foo\npwd'
 BUFFER='x'
 
 expected_region_highlight=(
-  '1 1 alias "issue #616"' # x
+  '1 1 alias' # x
 )

highlighters/main/test-data/alias-in-cmdsubst.zsh

@@ -0,0 +1,43 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2019 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+alias p='print -r --'
+
+BUFFER=$'s=$(p foo)'
+
+expected_region_highlight=(
+  '1 10 assign' # s=$(p foo)
+  '3 10 default' # $(p foo)
+  '3 10 command-substitution-unquoted' # $(p foo)
+  '3 4 command-substitution-delimiter-unquoted' # $(
+  '5 5 alias' # p
+  '7 9 default' # foo
+  '10 10 command-substitution-delimiter-unquoted' # )
+)

highlighters/main/test-data/alias-loop.zsh

@@ -28,12 +28,16 @@
 # vim: ft=zsh sw=2 ts=2 et
 # -------------------------------------------------------------------------------------------------
 
+function b() {} # beware of ALIAS_FUNC_DEF
 alias a=b b=c c=b
 
 BUFFER='a foo; :'
 
 expected_region_highlight=(
-  '1 1 unknown-token' # a (invalid alias loop)
+  # An alias is ineligible for expansion whilst it's being expanded.
+  # Therefore, the "b" in the expansion of the alias "c" is not considered
+  # as an alias.
+  '1 1 alias' # a
   '3 5 default' # foo
   '6 6 commandseparator' # ;
   '8 8 builtin' # :

highlighters/main/test-data/alias-to-dir.zsh

@@ -32,5 +32,5 @@ alias x=/
 BUFFER=$'x'
 
 expected_region_highlight=(
-  '1 1 unknown-token "issue #668"' # x (/)
+  '1 1 unknown-token' # x (/)
 )

highlighters/main/test-data/alias-to-dir1b.zsh

@@ -0,0 +1,37 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+setopt autocd
+alias x=/
+BUFFER=$'x'
+
+expected_region_highlight=(
+  '1 1 alias' # x
+)

highlighters/main/test-data/always2.zsh

@@ -32,7 +32,7 @@ BUFFER=$'{\nls\n} always { pwd }'
 
 expected_region_highlight=(
   '1 1 reserved-word' # {
-  '2 2 unknown-token' # \n
+  '2 2 commandseparator' # \n
   '3 4 command' # ls
   '5 5 commandseparator' # \n
   '6 6 reserved-word' # }

highlighters/main/test-data/arith-cmdsubst-mess.zsh

@@ -0,0 +1,46 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$': $((ls); (ls))'
+
+expected_region_highlight=(
+  '1 1 builtin' # :
+  '3 15 default' # $((ls); (ls))
+  '3 15 command-substitution-unquoted' # $((ls); (ls))
+  '3 4 command-substitution-delimiter-unquoted' # $(
+  '5 5 reserved-word' # (
+  '6 7 command' # ls
+  '8 8 reserved-word' # )
+  '9 9 commandseparator' # ;
+  '11 11 reserved-word' # (
+  '12 13 command' # ls
+  '14 14 reserved-word' # )
+  '15 15 command-substitution-delimiter-unquoted' # )
+)

highlighters/main/test-data/arith1.zsh

@@ -0,0 +1,37 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$': $(( 6 * 9 ))'
+
+expected_region_highlight=(
+  '1 1 builtin' # :
+  '3 14 default' # $(( 6 * 9 ))
+  '3 14 arithmetic-expansion' # $(( 6 * 9 ))
+)

highlighters/main/test-data/arith2.zsh

@@ -0,0 +1,38 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$': "$(( 6 * 9 ))"'
+
+expected_region_highlight=(
+  '1 1 builtin' # :
+  '3 16 default' # "$(( 6 * 9 ))"
+  '3 16 double-quoted-argument' # "$(( 6 * 9 ))"
+  '4 15 arithmetic-expansion' # $(( 6 * 9 ))
+)

highlighters/main/test-data/arithmetic-command-substitution.zsh

@@ -0,0 +1,42 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$': $(( $(echo 2) + 2 ))'
+
+expected_region_highlight=(
+  '1 1 builtin' # :
+  '3 22 default' # $(( $(echo 2) + 2 ))
+  '3 22 arithmetic-expansion' # $(( $(echo 2) + 2 ))
+  '7 15 command-substitution-quoted' # $(echo 2)
+  '7 8 command-substitution-delimiter-quoted' # $(
+  '9 12 builtin' # echo
+  '14 14 default' # 2
+  '15 15 command-substitution-delimiter' # )
+)

highlighters/main/test-data/arithmetic-doubled-parens.zsh

@@ -0,0 +1,37 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$': $(( ((42)) ))'
+
+expected_region_highlight=(
+  '1 1 builtin' # :
+  '3 15 default' # $(( ((42)) ))
+  '3 15 arithmetic-expansion' # $(( ((42)) ))
+)

highlighters/main/test-data/arithmetic-empty.zsh

@@ -0,0 +1,39 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$': "foo"$(())"bar"'
+
+expected_region_highlight=(
+  '1 1 builtin' # :
+  '3 17 default' # "foo"$(())"bar"
+  '3 7 double-quoted-argument' # "foo"
+  '8 12 arithmetic-expansion' # $(())
+  '13 17 double-quoted-argument' # "bar"
+)

highlighters/main/test-data/arithmetic-hist-expn.zsh

@@ -0,0 +1,38 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$': $(( \!\! ))'
+
+expected_region_highlight=(
+  '1 1 builtin' # :
+  '3 11 default' # $(( !! ))
+  '3 11 arithmetic-expansion' # $(( !! ))
+  '7 8 history-expansion "issue #713"' # !!
+)

highlighters/main/test-data/arithmetic-invalid-chars.zsh

@@ -0,0 +1,39 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$': $(( 0 * 1\'\'000 ))'
+
+expected_region_highlight=(
+  '1 1 builtin' # :
+  '3 19 default' # $(( 0 * 1\'\'000 ))
+  '3 19 arithmetic-expansion' # $(( 0 * 1\'\'000 ))
+  '12 12 unknown-token' # \'
+  '13 13 unknown-token' # \'
+)

highlighters/main/test-data/arithmetic-multiplication.zsh

@@ -0,0 +1,39 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$': foo*$(( 42 * 1729 ))*bar'
+
+expected_region_highlight=(
+  '1 1 builtin' # :
+  '3 26 default' # foo*$(( 42 * 1729 ))*bar
+  '6 6 globbing' # *
+  '7 22 arithmetic-expansion' # $(( 42 * 1729 ))
+  '23 23 globbing' # *
+)

highlighters/main/test-data/arithmetic-nested.zsh

@@ -0,0 +1,38 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$': $(( $(( 1 + 2 )) * 3 ))'
+
+expected_region_highlight=(
+  '1 1 builtin' # :
+  '3 25 default' # $(( $(( 1 + 2 )) * 3 ))
+  '3 25 arithmetic-expansion' # $(( $(( 1 + 2 )) * 3 ))
+  '7 18 arithmetic-expansion' # $(( 1 + 2 ))
+)

highlighters/main/test-data/arithmetic-quoted.zsh

@@ -0,0 +1,38 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$': "$(( 1 + 1 ))"'
+
+expected_region_highlight=(
+  '1 1 builtin' # :
+  '3 16 default' # "$(( 1 + 1 ))"
+  '3 16 double-quoted-argument' # "$(( 1 + 1 ))"
+  '4 15 arithmetic-expansion' # $(( 1 + 1 ))
+)

highlighters/main/test-data/arithmetic-unclosed.zsh

@@ -0,0 +1,37 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$': $(( 1'
+
+expected_region_highlight=(
+  '1 1 builtin' # :
+  '3 7 default' # $(( 1
+  '3 7 arithmetic-expansion' # $(( 1
+)

highlighters/main/test-data/arithmetic-unfinished.zsh

@@ -0,0 +1,41 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$': $(( 1729 )'
+
+expected_region_highlight=(
+  '1 1 builtin' # :
+  '3 12 default' # $(( 1729 )
+  '3 12 arithmetic-expansion' # $(( 1729 )
+)
+
+if [[ ${(z):-'$('} == '$( ' ]]; then # ignore zsh 5.0.8 bug
+  expected_region_highlight[2]='3 13 default' # $(( 1729 )
+fi

highlighters/main/test-data/array-cmdsep1.zsh

@@ -0,0 +1,45 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2019 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$'a=( foo | bar )'
+bar(){}
+
+expected_region_highlight=(
+  '1 3 assign' # a=(
+  '3 3 reserved-word' # (
+  '5 7 default' # foo
+  '9 9 unknown-token' # |
+  # zsh reports a parse error at this point.  Nevertheless, we test how we
+  # highlight the remainder of $BUFFER.  Currently we recover by treating the pipe
+  # as a command separator.  That's not the only reasonable behaviour, though; if
+  # we change the behaviour, we should adjust the following expectations accordingly.
+  '11 13 function' # bar
+  '15 15 unknown-token' # )
+)

highlighters/main/test-data/array-cmdsep2.zsh

@@ -0,0 +1,41 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2019 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$'a=( foo ; bar )'
+
+expected_region_highlight=(
+  '1 3 assign' # a=(
+  '3 3 reserved-word' # (
+  '5 7 default' # foo
+  '9 9 unknown-token' # ; (not commandseparator; see highlighter source code)
+  '11 13 default' # bar
+  '15 15 assign' # )
+  '15 15 reserved-word' # )
+)

highlighters/main/test-data/array-cmdsep3.zsh

@@ -0,0 +1,41 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2019 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$'a=( foo \n bar )'
+
+expected_region_highlight=(
+  '1 3 assign' # a=(
+  '3 3 reserved-word' # (
+  '5 7 default' # foo
+  '9 9 commandseparator' # \n
+  '11 13 default' # bar
+  '15 15 assign' # )
+  '15 15 reserved-word' # )
+)

highlighters/main/test-data/assign-append.zsh

@@ -31,7 +31,9 @@ BUFFER='a+=(lorem ipsum)'
 
 expected_region_highlight=(
   "1 4 assign" # a+=(
+  "4 4 reserved-word" # (
   "5 9 default" # lorem
   "11 15 default" # ipsum
   "16 16 assign" # )
+  "16 16 reserved-word" # )
 )

highlighters/main/test-data/assign-array.zsh

@@ -32,8 +32,10 @@ BUFFER='(A=(hello world))'
 expected_region_highlight=(
   "1 1 reserved-word" # (
   "2 4 assign" # A=(
+  "4 4 reserved-word" # (
   "5 9 default" # hello
   "11 15 default" # world
   "16 16 assign" # )
+  "16 16 reserved-word" # )
   "17 17 reserved-word" # )
 )

highlighters/main/test-data/assign-array2.zsh

@@ -31,8 +31,10 @@ BUFFER='A=(hello world) ls'
 
 expected_region_highlight=(
   "1 3 assign" # A=(
+  "3 3 reserved-word" # (
   "4 8 default" # hello
   "10 14 default" # world
   "15 15 assign" # )
+  "15 15 reserved-word" # )
   "17 18 command" # ls
 )

highlighters/main/test-data/assign-array3.zsh

@@ -31,9 +31,11 @@ BUFFER='A=(hello world) b=42'
 
 expected_region_highlight=(
   "1 3 assign" # A=(
+  "3 3 reserved-word" # (
   "4 8 default" # hello
   "10 14 default" # world
   "15 15 assign" # )
+  "15 15 reserved-word" # )
   "17 20 assign" # b=42
   "19 20 default" # 42
 )

highlighters/main/test-data/assign-invalid-command.zsh

@@ -0,0 +1,37 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$'x=y nosuchcommand'
+
+expected_region_highlight=(
+  '1 3 assign' # x=y
+  '3 3 default' # y
+  '5 17 unknown-token' # nosuchcommand
+)

highlighters/main/test-data/assign-quoted-cmdsubst.zsh

@@ -0,0 +1,45 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2018 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$'x="$(ls x y z)"'
+
+expected_region_highlight=(
+  '1 15 assign' # x="$(ls x y z)"
+  '3 15 default' # "$(ls x y z)"
+  '3 3 double-quoted-argument' # "
+  '15 15 double-quoted-argument' # "
+  '4 14 command-substitution-quoted' # $(ls x y z)
+  '4 5 command-substitution-delimiter-quoted' # $(
+  '6 7 command' # ls
+  '9 9 default' # x
+  '11 11 default' # y
+  '13 13 default' # z
+  '14 14 command-substitution-delimiter-quoted' # )
+)

highlighters/main/test-data/assign.zsh

@@ -33,8 +33,10 @@ expected_region_highlight=(
   "1 3 assign" # A=1
   "3 3 default" # 1
   "5 7 assign" # b=(
+  "7 7 reserved-word" # (
   "8 12 default" # "foo"
   "8 12 double-quoted-argument" # "foo"
   "14 16 default" # bar
   "17 17 assign" # )
+  "17 17 reserved-word" # )
 )

highlighters/main/test-data/assignment-before-resword1.zsh

@@ -0,0 +1,40 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2017 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$'foo=bar { :; }'
+
+expected_region_highlight=(
+  '1 7 assign' # foo=bar
+  '5 7 default' # bar
+  '9 9 unknown-token' # {
+  '11 11 builtin' # :
+  '12 12 commandseparator' # ;
+  '14 14 reserved-word' # }
+)

highlighters/main/test-data/assignment-before-resword2.zsh

@@ -0,0 +1,40 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2017 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$'foo=bar ( :; )'
+
+expected_region_highlight=(
+  '1 7 assign' # foo=bar
+  '5 7 default' # bar
+  '9 9 unknown-token' # (
+  '11 11 builtin' # :
+  '12 12 commandseparator' # ;
+  '14 14 unknown-token' # )
+)

highlighters/main/test-data/assignment-before-resword3.zsh

@@ -0,0 +1,37 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2017 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$'foo=bar (( foo ))'
+
+expected_region_highlight=(
+  '1 7 assign' # foo=bar
+  '5 7 default' # bar
+  '9 17 unknown-token' # (( foo ))
+)

highlighters/main/test-data/assignment-before-resword4.zsh

@@ -0,0 +1,40 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2017 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$'foo=bar [[ -n foo ]]'
+
+expected_region_highlight=(
+  '1 7 assign' # foo=bar
+  '5 7 default' # bar
+  '9 10 unknown-token' # [[
+  '12 13 single-hyphen-option' # -n
+  '15 17 default' # foo
+  '19 20 reserved-word' # ]]
+)

highlighters/main/test-data/assignment-before-resword5.zsh

@@ -0,0 +1,38 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2017 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$'foo=bar \! :'
+
+expected_region_highlight=(
+  '1 7 assign' # foo=bar
+  '5 7 default' # bar
+  '9 9 unknown-token' # \!
+  '11 11 builtin' # :
+)

highlighters/main/test-data/backslash-continuation2.zsh

@@ -0,0 +1,37 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$'echo foo\\\nbar"baz"'
+
+expected_region_highlight=(
+  '1 4 builtin' # echo
+  '6 18 default "issue #705"' # foo\\\nbar"baz"
+  '14 18 double-quoted-argument "issue #705"' # "baz"
+)

highlighters/main/test-data/bang-assign-array.zsh

@@ -32,9 +32,11 @@ BUFFER=$'foo=(bar abaz) \! ls'
 
 expected_region_highlight=(
   '1 5 assign' # foo=(
+  '5 5 reserved-word' # (
   '6 8 default' # bar
   '10 13 default' # abaz
   '14 14 assign' # )
+  '14 14 reserved-word' # )
   '16 16 unknown-token' # \!
   '18 19 command' # ls
 )

highlighters/main/test-data/cdpath-abspath.zsh

@@ -0,0 +1,38 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+cdpath=( $PWD )
+mkdir foo foo/bar
+
+BUFFER="/foo"
+
+expected_region_highlight=(
+  '1 4 unknown-token' # x (/)
+)

highlighters/main/test-data/comment-followed.zsh

@@ -28,11 +28,12 @@
 # vim: ft=zsh sw=2 ts=2 et
 # -------------------------------------------------------------------------------------------------
 
+# see alias-comment1.zsh
 setopt interactivecomments
 BUFFER=$'# foo\ntrue'
 
 expected_region_highlight=(
   '1 5 comment' # # foo
-  '6 6 commandseparator "issue #501"' # \n
+  '6 6 commandseparator' # \n
   '7 10 builtin' # true
 )

highlighters/main/test-data/dinbrack1.zsh

@@ -0,0 +1,41 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$'[[ foo && bar || baz ]]'
+
+expected_region_highlight=(
+  '1 2 reserved-word' # [[
+  '4 6 default' # foo
+  '8 9 default' # &&
+  '11 13 default' # bar
+  '15 16 default' # ||
+  '18 20 default' # baz
+  '22 23 reserved-word' # ]]
+)

highlighters/main/test-data/empty-command-newline.zsh

@@ -34,6 +34,6 @@ BUFFER=$':;\n:'
 expected_region_highlight=(
   '1 1 builtin' # :
   '2 2 commandseparator' # ;
-  '3 3 commandseparator "issue #616"' # \n
+  '3 3 commandseparator' # \n
   '4 4 builtin' # :
 )

highlighters/main/test-data/equals1.zsh

@@ -0,0 +1,36 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$': =ls'
+
+expected_region_highlight=(
+  '1 1 builtin' # :
+  '3 5 path' # =ls
+)

highlighters/main/test-data/equals2.zsh

@@ -0,0 +1,38 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+unsetopt equals
+
+BUFFER=$': =nosuchcommand'
+
+expected_region_highlight=(
+  '1 1 builtin' # :
+  '3 16 default' # =nosuchcommand
+)

highlighters/main/test-data/equals3.zsh

@@ -0,0 +1,36 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$': =nosuchcommand'
+
+expected_region_highlight=(
+  '1 1 builtin' # :
+  '3 16 unknown-token' # =nosuchcommand
+)

highlighters/main/test-data/equals4.zsh

@@ -0,0 +1,36 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$': ='
+
+expected_region_highlight=(
+  '1 1 builtin' # :
+  '3 3 default' # =
+)

highlighters/main/test-data/fd-target-not-filename.zsh

@@ -0,0 +1,40 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+touch 2
+
+BUFFER=$'echo foo>&2'
+
+expected_region_highlight=(
+  '1 4 builtin' # echo
+  '6 8 default' # foo
+  '9 10 redirection' # >&
+  '11 11 numeric-fd' # 2 (not path)
+)

highlighters/main/test-data/global-alias1.zsh

@@ -0,0 +1,38 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+alias -g foo=bar
+
+BUFFER=$'foo foo'
+
+expected_region_highlight=(
+  '1 3 global-alias' # foo
+  '5 7 global-alias' # foo
+)

highlighters/main/test-data/history-double-quoted-followed.zsh

@@ -0,0 +1,39 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=': !!= "!!="'
+
+expected_region_highlight=(
+  '1 1 builtin' # :
+  '3 4 history-expansion "issue #713"' # !!
+  '7 11 default' # "!!="
+  '7 11 double-quoted-argument' # "!!="
+  '8 9 history-expansion "issue #713' # !!
+)

highlighters/main/test-data/meta-no-eval1.zsh

@@ -0,0 +1,49 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$': $(kill -9 $$) ${:-$(kill -9 $$)}'
+
+expected_region_highlight=(
+  '1 1 builtin' # :
+  '3 15 default' # $(kill -9 $$)
+  '3 15 command-substitution-unquoted' # $(kill -9 $$)
+  '3 4 command-substitution-delimiter-unquoted' # $(
+  '5 8 builtin' # kill
+  '10 11 single-hyphen-option' # -9
+  '13 14 default' # $$
+  '15 15 command-substitution-delimiter-unquoted' # )
+  '17 34 default' # ${:-$(kill -9 $$)}
+  '21 33 command-substitution-unquoted' # $(kill -9 $$)
+  '21 22 command-substitution-delimiter-unquoted' # $(
+  '23 26 builtin' # kill
+  '28 29 single-hyphen-option' # -9
+  '31 32 default' # $$
+  '33 33 command-substitution-delimiter-unquoted' # )
+)

highlighters/main/test-data/meta-no-eval2.zsh

@@ -0,0 +1,40 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+# We aren't testing how this is highlighted; we're testing that it's not
+# evaluated.  If it gets evaluated, the test suite will die.
+BUFFER=$': /(e*exit 42*)'
+
+expected_region_highlight=(
+  '1 1 builtin' # :
+  '3 15 default' # /(e*exit 42*)
+  '6 6 globbing' # *
+  '14 14 globbing' # *
+)

highlighters/main/test-data/multiline-array-assignment1.zsh

@@ -32,8 +32,10 @@ BUFFER=$'foo=(\nbar) env'
 
 expected_region_highlight=(
   '1 5 assign' # foo=(
+  '5 5 reserved-word' # (
   '6 6 commandseparator' # \n
   '7 9 default' # bar
   '10 10 assign' # )
-  '12 14 command' # env
+  '10 10 reserved-word' # )
+  '12 14 precommand' # env
 )

highlighters/main/test-data/multios-negates-globbing.zsh

@@ -0,0 +1,39 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2018 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+unsetopt multios
+
+BUFFER=$'cat < *'
+
+expected_region_highlight=(
+  '1 3 command' # cat
+  '5 5 redirection' # <
+  '7 7 default' # * - not globbing
+)

highlighters/main/test-data/multios-negates-globbing2.zsh

@@ -0,0 +1,38 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2018 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$'cat < *'
+
+expected_region_highlight=(
+  '1 3 command' # cat
+  '5 5 redirection' # <
+  '7 7 default' # *
+  '7 7 globbing' # *
+)

highlighters/main/test-data/noglob-always.zsh

@@ -39,6 +39,7 @@ expected_region_highlight=(
   '19 24 reserved-word' # always
   '26 26 reserved-word' # {
   '28 31 builtin' # echo
-  '33 33 globbing "issue #577"' # *
+  '33 33 default' # *
+  '33 33 globbing' # *
   '35 35 reserved-word' # }
 )

highlighters/main/test-data/null-exec.zsh

@@ -34,5 +34,5 @@ expected_region_highlight=(
   '1 4 precommand' # exec
   '6 6 redirection' # >
   '7 15 path' # /dev/null
-  '16 16 commandseparator "issue #676"' # ;
+  '16 16 commandseparator' # ;
 )

highlighters/main/test-data/opt-shwordsplit1.zsh

@@ -0,0 +1,40 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+setopt shwordsplit
+local EDITOR='ed -s'
+
+ed() { command ed "$@" }
+
+BUFFER=$'$EDITOR'
+
+expected_region_highlight=(
+  '1 7 function "issue #687"' # $EDITOR
+)

highlighters/main/test-data/optimized-cmdsubst-input.zsh

@@ -0,0 +1,43 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2018 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+# See getoutput() and getoutputfile() in zsh's C source code.
+
+BUFFER=$': $(<*)'
+
+expected_region_highlight=(
+  '1 1 builtin' # :
+  '3 7 default' # $(<*)
+  '3 7 command-substitution-unquoted' # $(<*)
+  '3 4 command-substitution-delimiter-unquoted' # $(
+  '5 5 redirection' # <
+  '6 6 default' # * - not globbing!
+  '7 7 command-substitution-delimiter-unquoted' # )
+)

highlighters/main/test-data/param-positional-in-array-append.zsh

@@ -0,0 +1,39 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+# This used to be an infinite loop.
+
+BUFFER=$'l+=( $1'
+
+expected_region_highlight=(
+  '1 4 assign' # l+=(
+  '4 4 reserved-word' # (
+  '6 7 default' # $1
+)

highlighters/main/test-data/parameter-elision-command-word.zsh

@@ -0,0 +1,36 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER='$x ls'
+
+expected_region_highlight=(
+  '1 2 comment' # $x
+  '4 5 command' # ls
+)

highlighters/main/test-data/parameter-to-global-alias.zsh

@@ -0,0 +1,38 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+alias -g x=y
+local s=x
+
+BUFFER=$'$s'
+
+expected_region_highlight=(
+  '1 2 unknown-token' # $s
+)

highlighters/main/test-data/path-dollared-word3.zsh

@@ -34,7 +34,7 @@
 BUFFER='$PWD; ${PWD}'
 
 expected_region_highlight=(
-  "1 4 path" # $PWD
+  "1 4 unknown-token" # $PWD (without AUTO_CD)
   "5 5 commandseparator" # ;
-  "7 12 path" # ${PWD}
+  "7 12 path_prefix" # ${PWD}
 )

highlighters/main/test-data/path-dollared-word3b.zsh

@@ -0,0 +1,38 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+setopt autocd
+BUFFER=$'$PWD; ${PWD}'
+
+expected_region_highlight=(
+  '1 4 autodirectory' # $PWD
+  '5 5 commandseparator' # ;
+  '7 12 autodirectory' # ${PWD}
+)

highlighters/main/test-data/path-mixed-quoting.zsh

@@ -0,0 +1,39 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+touch foo
+
+BUFFER=$': \'f\'oo'
+
+expected_region_highlight=(
+  '1 1 builtin' # :
+  '3 7 path' # \'f\'oo
+  '3 5 single-quoted-argument' # \'f\'
+)

highlighters/main/test-data/path_prefix3.zsh

@@ -30,7 +30,7 @@
 # Assumes that '/bin/sh' exists and '/bin/s' does not exist.
 # Related to path_prefix.zsh
 
-PREBUFFER='ls \'
+PREBUFFER=$'ls \\\n'
 BUFFER='/bin/s'
 
 expected_region_highlight=(

highlighters/main/test-data/plain-file-in-command-position.zsh

@@ -0,0 +1,39 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+touch foo
+chmod -x foo
+BUFFER=$'./foo; ./foo'
+
+expected_region_highlight=(
+  '1 5 unknown-token' # ./foo (in middle)
+  '6 6 commandseparator' # ;
+  '8 12 unknown-token' # ./foo (at end)
+)

highlighters/main/test-data/precommand-killing2.zsh

@@ -35,5 +35,5 @@ BUFFER='sudo -e /does/not/exist'
 expected_region_highlight=(
   '1 4 precommand' # sudo
   '6 7 single-hyphen-option' # -e
-  '9 23 normal "issue #678"' # /does/not/exist
+  '9 23 default' # /does/not/exist
 )

highlighters/main/test-data/precommand-then-assignment.zsh

@@ -0,0 +1,37 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2019 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$'nice x=y ls'
+
+expected_region_highlight=(
+  '1 4 precommand' # nice
+  '6 8 unknown-token "issue #641.5"' # x=y
+  '10 11 default "issue #641.5 (fallout)"' # ls
+)

highlighters/main/test-data/precommand-uninstalled.zsh

@@ -0,0 +1,44 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+# Simulate sudo not being installed.
+#
+# The 'hash' step is because, if sudo _really_ isn't installed, 'unhash sudo'
+# would error out and break the test.
+hash sudo=/usr/bin/env && unhash sudo
+
+local PATH
+
+BUFFER=$'sudo ls'
+
+expected_region_highlight=(
+  '1 4 unknown-token' # sudo
+  '6 7 default' # ls - not 'command', since sudo isn't installed
+)

highlighters/main/test-data/redirection-from-param.zsh

@@ -0,0 +1,40 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+touch file
+local fn=$PWD/file
+
+BUFFER=$'<$fn cat'
+
+expected_region_highlight=(
+  '1 1 redirection' # <
+  '2 4 path' # $fn
+  '6 8 command' # cat
+)

highlighters/main/test-data/redirection-inhibits-elision.zsh

@@ -0,0 +1,38 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$'<$foo cat cat'
+
+expected_region_highlight=(
+  '1 1 redirection' # <
+  '2 5 default' # $foo
+  '7 9 command' # cat
+  '11 13 default' # cat
+)

highlighters/main/test-data/redirection-special-cases.zsh

@@ -0,0 +1,44 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2019 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+# See xpandredir() in the zsh source.
+
+BUFFER=$'cat <&p; exec {myfd}>&-'
+
+expected_region_highlight=(
+  '1 3 command' # cat
+  '5 6 redirection' # <&
+  '7 7 redirection' # p
+  '8 8 commandseparator' # ;
+  '10 13 precommand' # exec
+  '15 20 named-fd' # {myfd}
+  '21 22 redirection' # >&
+  '23 23 redirection' # -
+)

highlighters/pattern/pattern-highlighter.zsh

@@ -54,7 +54,7 @@ _zsh_highlight_pattern_highlighter_loop()
   local -a match mbegin mend
   local MATCH; integer MBEGIN MEND
   if [[ "$buf" == (#b)(*)(${~pat})* ]]; then
-    region_highlight+=("$((mbegin[2] - 1)) $mend[2] $ZSH_HIGHLIGHT_PATTERNS[$pat]")
+    region_highlight+=("$((mbegin[2] - 1)) $mend[2] $ZSH_HIGHLIGHT_PATTERNS[$pat], memo=zsh-syntax-highlighting")
     "$0" "$match[1]" "$pat"; return $?
   fi
 }

highlighters/regexp/regexp-highlighter.zsh

@@ -55,7 +55,7 @@ _zsh_highlight_regexp_highlighter_loop()
   local -a match mbegin mend
   while true; do
     [[ "$buf" =~ "$pat" ]] || return;
-    region_highlight+=("$((MBEGIN - 1 + OFFSET)) $((MEND + OFFSET)) $ZSH_HIGHLIGHT_REGEXP[$pat]")
+    region_highlight+=("$((MBEGIN - 1 + OFFSET)) $((MEND + OFFSET)) $ZSH_HIGHLIGHT_REGEXP[$pat], memo=zsh-syntax-highlighting")
     buf="$buf[$(($MEND+1)),-1]"
     OFFSET=$((MEND+OFFSET));
   done

release.md

@@ -8,7 +8,8 @@
     (easiest to check travis: https://travis-ci.org/zsh-users/zsh-syntax-highlighting/)
 - Update changelog.md
   `tig --abbrev=12  --abbrev-commit 0.4.1..upstream/master`
-- Remove `-dev` suffix from `./.version`;
+- Make sure there are no local commits and that `git status` is clean;
+  Remove `-dev` suffix from `./.version`;
   Commit that using `git commit -m "Tag version $(<.version)." .version`;
   Tag it using `git tag -s -m "Tag version $(<.version)" $(<.version)`;
   Increment `./.version` and restore the `-dev` suffix;

tests/README.md

@@ -7,6 +7,9 @@ The tests harness expects the highlighter directory to contain a `test-data`
 directory with test data files.
 See the [main highlighter](../highlighters/main/test-data) for examples.
 
+Tests should set the following variables:
+
+1.
 Each test should define the string `$BUFFER` that is to be highlighted and the
 array parameter `$expected_region_highlight`.
 The value of that parameter is a list of strings of the form  `"$i $j $style"`.
@@ -16,15 +19,26 @@ that is, `$i` and `$j` specify a range, 1-indexed, inclusive of both endpoints.
 `$style` is a key of `$ZSH_HIGHLIGHT_STYLES`.
 If `$todo` exists, the test point is marked as TODO (the failure of that test
 point will not fail the test), and `$todo` is used as the explanation.
+
+2. 
 If a test sets `$skip_test` to a non-empty string, the test will be skipped
 with the provided string as the reason.
+
+3. 
+If a test sets `$fail_test` to a non-empty string, the test will be skipped
+with the provided string as the reason.
+
+4.
 If a test sets `unsorted=1` the order of highlights in `$expected_region_highlight`
 need not match the order in `$region_highlight`.
 
+5.
 Normally, tests fail if `$expected_region_highlight` and `$region_highlight`
-have different numbers of elements.  Tests may set `$expected_mismatch` to an
+have different numbers of elements.  To mark this check as expected to fail,
-explanation string (like `$todo`) to avoid this and mark the cardinality check
+tests may set `$expected_mismatch` to an explanation string (like `$todo`);
-as todo.
+this is useful when the only difference between actual and expected is that actual
+has some additional, superfluous elements.  This check is skipped if the
+`$todo` component is present in any regular test point.
 
 **Note**: `$region_highlight` uses the same `"$i $j $style"` syntax but
 interprets the indexes differently.

tests/edit-failed-tests

@@ -32,7 +32,7 @@ type perl sponge >/dev/null || { print -ru2 -- "$0: This script requires perl(1)
 
 local editor=( "${(@Q)${(z)${VISUAL:-${EDITOR:-vi}}}}" )
 () { 
-  > "$2" perl -nE '$highlighter = $1 if /^Running test (\S*)/; say "highlighters/${highlighter}/test-data/$1.zsh" if /^# (\S*)/' "$1"
+  > "$2" perl -nE '$highlighter = $1 if /^Running test (\S*)/; say "highlighters/${highlighter}/test-data/$1.zsh" if /^## (\S*)/' "$1"
   >>"$2" echo ""
   >>"$2" cat <"$1"
   "${editor[@]}" -- "$2"

tests/generate.zsh

@@ -31,6 +31,9 @@
 emulate -LR zsh
 setopt localoptions extendedglob
 
+# Required for add-zle-hook-widget.
+zmodload zsh/zle
+
 # Argument parsing.
 if (( $# * $# - 7 * $# + 12 )) || [[ $1 == -* ]]; then
   print -r -- >&2 "$0: usage: $0 BUFFER HIGHLIGHTER BASENAME [PREAMBLE]"

tests/test-highlighting.zsh

@@ -31,6 +31,9 @@
 
 setopt NO_UNSET WARN_CREATE_GLOBAL
 
+# Required for add-zle-hook-widget.
+zmodload zsh/zle
+
 local -r root=${0:h:h}
 local -a anon_argv; anon_argv=("$@")
 
@@ -95,9 +98,9 @@ ZSH_HIGHLIGHT_HIGHLIGHTERS=($1)
 typeset_p() {
   for 1 ; do
     if [[ ${(tP)1} == *array* ]]; then
-      print -r -- "$1=( ${(@q-P)1} )"
+      print -r -- "$1=( ${(@qqqqP)1} )"
     else
-      print -r -- "$1=${(q-P)1}"
+      print -r -- "$1=${(qqqqP)1}"
     fi
   done
 }
@@ -105,7 +108,7 @@ typeset_p() {
 # Escape # as ♯ and newline as ↵ they are illegal in the 'description' part of TAP output
 # The string to escape is «"$@"»; the result is returned in $REPLY.
 tap_escape() {
-  local s="$@"
+  local s="${(j. .)@}"
   REPLY="${${s//'#'/♯}//$'\n'/↵}"
 }
 
@@ -118,24 +121,44 @@ run_test_internal() {
   builtin cd -q -- "$tests_tempdir" || { echo >&2 "Bail out! On ${(qq)1}: cd failed: $?"; return 1 }
 
   # Load the data and prepare checking it.
-  local BUFFER CURSOR MARK PENDING PREBUFFER REGION_ACTIVE WIDGET REPLY skip_test unsorted=0
+  local BUFFER CURSOR MARK PENDING PREBUFFER REGION_ACTIVE WIDGET REPLY skip_test fail_test unsorted=0
   local expected_mismatch
+  local skip_mismatch
   local -a expected_region_highlight region_highlight
 
-  . "$srcdir"/"$1"
+  local ARG="$1"
+  local RETURN=""
+  () {
+    setopt localoptions
 
-  (( $#skip_test )) && { print -r -- "1..0 # SKIP $skip_test"; return; }
+    # WARNING: The remainder of this anonymous function will run with the test's options in effect
+    if { ! . "$srcdir"/"$ARG" } || (( $#fail_test )); then
+      print -r -- "1..1"
+      print -r -- "## ${ARG:t:r}"
+      tap_escape $fail_test; fail_test=$REPLY
+      print -r -- "not ok 1 - failed setup: $fail_test"
+      return ${RETURN:=0}
+    fi
 
-  # Check the data declares $PREBUFFER or $BUFFER.
+    (( $#skip_test )) && {
-  [[ -z $PREBUFFER && -z $BUFFER ]] && { echo >&2 "Bail out! On ${(qq)1}: Either 'PREBUFFER' or 'BUFFER' must be declared and non-blank"; return 1; }
+      print -r -- "1..0 # SKIP $skip_test"
-  # Check the data declares $expected_region_highlight.
+      print -r -- "## ${ARG:t:r}"
-  (( $+expected_region_highlight == 0 )) && { echo >&2 "Bail out! On ${(qq)1}: 'expected_region_highlight' is not declared."; return 1; }
+      return ${RETURN:=0}
+    }
 
-  # Set sane defaults for ZLE variables
+    # Check the data declares $PREBUFFER or $BUFFER.
-  : ${CURSOR=$#BUFFER} ${PENDING=0} ${WIDGET=z-sy-h-test-harness-test-widget}
+    [[ -z $PREBUFFER && -z $BUFFER ]] && { echo >&2 "Bail out! On ${(qq)ARG}: Either 'PREBUFFER' or 'BUFFER' must be declared and non-blank"; return ${RETURN:=1}; }
+    [[ $PREBUFFER == (''|*$'\n') ]] || { echo >&2 "Bail out! On ${(qq)ARG}: PREBUFFER=${(qqqq)PREBUFFER} doesn't end with a newline"; return ${RETURN:=1}; }
 
-  # Process the data.
+    # Set sane defaults for ZLE variables
-  _zsh_highlight
+    : ${CURSOR=$#BUFFER} ${PENDING=0} ${WIDGET=z-sy-h-test-harness-test-widget}
+
+    # Process the data.
+    _zsh_highlight
+  }; [[ -z $RETURN ]] || return $RETURN
+  unset ARG
+
+  integer print_expected_and_actual=0
 
   if (( unsorted )); then
     region_highlight=("${(@n)region_highlight}")
@@ -144,7 +167,7 @@ run_test_internal() {
 
   # Print the plan line, and some comments for human readers
   echo "1..$(( $#expected_region_highlight + 1))"
-  echo "## ${1:t:r}"
+  echo "## ${1:t:r}" # note: tests/edit-failed-tests looks for the "##" emitted by this line
   [[ -n $PREBUFFER ]] && printf '# %s\n' "$(typeset_p PREBUFFER)"
   [[ -n $BUFFER ]] && printf '# %s\n' "$(typeset_p BUFFER)"
 
@@ -153,9 +176,14 @@ run_test_internal() {
     local -a expected_highlight_zone; expected_highlight_zone=( ${(z)expected_region_highlight[i]} )
     integer exp_start=$expected_highlight_zone[1] exp_end=$expected_highlight_zone[2]
     local todo=
-    (( $+expected_highlight_zone[4] )) && todo="# TODO $expected_highlight_zone[4]"
+    if (( $+expected_highlight_zone[4] )); then
+      todo="# TODO $expected_highlight_zone[4]"
+      skip_mismatch="cardinality check disabled whilst regular test points are expected to fail"
+    fi
     if ! (( $+region_highlight[i] )); then
-      print -r -- "not ok $i - unmatched expectation ($exp_start $exp_end $expected_highlight_zone[3])"
+      print -r -- "not ok $i - unmatched expectation ($exp_start $exp_end $expected_highlight_zone[3])" \
+         "${skip_mismatch:+"# TODO ${(qqq)skip_mismatch}"}"
+      if [[ -z $skip_mismatch ]]; then (( ++print_expected_and_actual )); fi
       continue
     fi
     local -a highlight_zone; highlight_zone=( ${(z)region_highlight[i]} )
@@ -165,9 +193,10 @@ run_test_internal() {
     if
       [[ $start != $exp_start ]] ||
       [[ $end != $exp_end ]] ||
-      [[ $highlight_zone[3] != $expected_highlight_zone[3] ]]
+      [[ ${highlight_zone[3]%,} != ${expected_highlight_zone[3]} ]] # remove the comma that's before the memo field
     then
       print -r -- "not ok $i - $desc - expected ($exp_start $exp_end ${(qqq)expected_highlight_zone[3]}), observed ($start $end ${(qqq)highlight_zone[3]}). $todo"
+      if [[ -z $todo ]]; then (( ++print_expected_and_actual )); fi
     else
       print -r -- "ok $i - $desc${todo:+ - }$todo"
     fi
@@ -179,14 +208,43 @@ run_test_internal() {
     unset desc
   done
 
-  if (( $#expected_region_highlight == $#region_highlight )); then
+  # If both $skip_mismatch and $expected_mismatch are set, that means the test
-    print -r -- "ok $i - cardinality check" "${expected_mismatch:+"# TODO ${(qqq)expected_mismatch}"}"
+  # has some XFail test points, _and_ explicitly sets $expected_mismatch as
+  # well.  Explicit settings should have priority, so we ignore $skip_mismatch
+  # if $expected_mismatch is set.
+  if [[ -n $skip_mismatch && -z $expected_mismatch ]]; then
+    tap_escape $skip_mismatch; skip_mismatch=$REPLY
+    print "ok $i - cardinality check" "# SKIP $skip_mismatch"
   else
-    local details
+    local todo
-    details+="have $#expected_region_highlight expectations and $#region_highlight region_highlight entries: "
+    if [[ -n $expected_mismatch ]]; then
-    details+="«$(typeset_p expected_region_highlight)» «$(typeset_p region_highlight)»"
+      tap_escape $expected_mismatch; expected_mismatch=$REPLY
-    tap_escape $details; details=$REPLY
+      todo="# TODO $expected_mismatch"
-    print -r -- "not ok $i - $details" "${expected_mismatch:+"# TODO ${(qqq)expected_mismatch}"}"
+    fi
+    if (( $#expected_region_highlight == $#region_highlight )); then
+      print -r -- "ok $i - cardinality check${todo:+ - }$todo"
+    else
+      local details
+      details+="have $#expected_region_highlight expectations and $#region_highlight region_highlight entries: "
+      details+="«$(typeset_p expected_region_highlight)» «$(typeset_p region_highlight)»"
+      tap_escape $details; details=$REPLY
+      print -r -- "not ok $i - cardinality check - $details${todo:+ - }$todo"
+      if [[ -z $todo ]]; then (( ++print_expected_and_actual )); fi
+    fi
+  fi
+  if (( print_expected_and_actual )); then
+      () {
+        local -a left_column right_column
+        left_column=( "expected_region_highlight" "${(qq)expected_region_highlight[@]}" )
+        right_column=( "region_highlight" "${(qq)region_highlight[@]}" )
+        integer difference=$(( $#right_column - $#left_column ))
+        repeat $difference do left_column+=(.); done
+        paste \
+          =(print -rC1 -- $left_column) \
+          =(print -rC1 -- $right_column) \
+          | if type column >/dev/null; then column -t -s $'\t'; else cat; fi \
+          | sed 's/^/# /'
+      }
   fi
 }
 

tests/test-perfs.zsh

@@ -29,6 +29,9 @@
 # -------------------------------------------------------------------------------------------------
 
 
+# Required for add-zle-hook-widget.
+zmodload zsh/zle
+
 # Check an highlighter was given as argument.
 [[ -n "$1" ]] || {
   echo >&2 "Bail out! You must provide the name of a valid highlighter as argument."
@@ -48,6 +51,7 @@
 }
 
 # Load the main script.
+typeset -a region_highlight
 . ${0:h:h}/zsh-syntax-highlighting.zsh
 
 # Activate the highlighter.
@@ -62,8 +66,6 @@ run_test_internal() {
   local srcdir="$PWD"
   builtin cd -q -- "$tests_tempdir" || { echo >&2 "Bail out! cd failed: $?"; return 1 }
 
-  echo -n "# ${1:t:r}: "
-
   # Load the data and prepare checking it.
   PREBUFFER= BUFFER= ;
   . "$srcdir"/"$1"
@@ -71,9 +73,8 @@ run_test_internal() {
   # Check the data declares $PREBUFFER or $BUFFER.
   [[ -z $PREBUFFER && -z $BUFFER ]] && { echo >&2 "Bail out! Either 'PREBUFFER' or 'BUFFER' must be declared and non-blank"; return 1; }
 
-  # Measure the time taken by _zsh_highlight.
+  # Set $? for _zsh_highlight
-  TIMEFMT="%*Es"
+  true && _zsh_highlight
-  time (BUFFER="$BUFFER" && _zsh_highlight)
 }
 
 run_test() {
@@ -93,9 +94,10 @@ run_test() {
 
 # Process each test data file in test data directory.
 local data_file
-for data_file in ${0:h:h}/highlighters/$1/test-data/*.zsh; do
+TIMEFMT="%*Es"
+{ time (for data_file in ${0:h:h}/highlighters/$1/test-data/*.zsh; do
   run_test "$data_file"
   (( $pipestatus[1] )) && exit 2
-done
+done) } 2>&1 || exit $?
 
 exit 0

tests/test-zprof.zsh

@@ -0,0 +1,78 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2010-2015 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+# Load the main script.
+typeset -a region_highlight
+. ${0:h:h}/zsh-syntax-highlighting.zsh
+
+# Activate the highlighter.
+ZSH_HIGHLIGHT_HIGHLIGHTERS=(main)
+
+source_file=0.7.1:highlighters/$1/$1-highlighter.zsh
+
+# Runs a highlighting test
+# $1: data file
+run_test_internal() {
+  setopt interactivecomments
+
+  local -a highlight_zone
+
+  local tests_tempdir="$1"; shift
+  local srcdir="$PWD"
+  builtin cd -q -- "$tests_tempdir" || { echo >&2 "Bail out! cd failed: $?"; return 1 }
+
+  # Load the data and prepare checking it.
+  PREBUFFER=
+  BUFFER=$(cd -- "$srcdir" && git cat-file blob $source_file)
+  expected_region_highlight=()
+
+  zmodload zsh/zprof
+  zprof -c
+  # Set $? for _zsh_highlight
+  true && _zsh_highlight
+  zprof
+}
+
+run_test() {
+  # Do not combine the declaration and initialization: «local x="$(false)"» does not set $?.
+  local __tests_tempdir
+  __tests_tempdir="$(mktemp -d)" && [[ -d $__tests_tempdir ]] || {
+    echo >&2 "Bail out! mktemp failed"; return 1
+  }
+  typeset -r __tests_tempdir # don't allow tests to override the variable that we will 'rm -rf' later on
+
+  {
+    (run_test_internal "$__tests_tempdir" "$@")
+  } always {
+    rm -rf -- "$__tests_tempdir"
+  }
+}
+
+run_test

zsh-syntax-highlighting.zsh

@@ -1,5 +1,5 @@
 # -------------------------------------------------------------------------------------------------
-# Copyright (c) 2010-2016 zsh-syntax-highlighting contributors
+# Copyright (c) 2010-2020 zsh-syntax-highlighting contributors
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification, are permitted
@@ -29,8 +29,8 @@
 
 # First of all, ensure predictable parsing.
 typeset zsh_highlight__aliases="$(builtin alias -Lm '[^+]*')"
-# In zsh <= 5.2, `alias -L` emits aliases that begin with a plus sign ('alias -- +foo=42')
+# In zsh <= 5.2, aliases that begin with a plus sign ('alias -- +foo=42')
-# them without a '--' guard, so they don't round trip.
+# are emitted by `alias -L` without a '--' guard, so they don't round trip.
 #
 # Hence, we exclude them from unaliasing:
 builtin unalias -m '[^+]*'
@@ -49,6 +49,52 @@ if true; then
   fi
 fi
 
+# This function takes a single argument F and returns True iff F is an autoload stub.
+_zsh_highlight__function_is_autoload_stub_p() {
+  if zmodload -e zsh/parameter; then
+    #(( ${+functions[$1]} )) &&
+    [[ "$functions[$1]" == *"builtin autoload -X"* ]]
+  else
+    #[[ $(type -wa -- "$1") == *'function'* ]] &&
+    [[ "${${(@f)"$(which -- "$1")"}[2]}" == $'\t'$histchars[3]' undefined' ]]
+  fi
+  # Do nothing here: return the exit code of the if.
+}
+
+# Return True iff the argument denotes a function name.
+_zsh_highlight__is_function_p() {
+  if zmodload -e zsh/parameter; then
+    (( ${+functions[$1]} ))
+  else
+    [[ $(type -wa -- "$1") == *'function'* ]]
+  fi
+}
+
+# This function takes a single argument F and returns True iff F denotes the
+# name of a callable function.  A function is callable if it is fully defined
+# or if it is marked for autoloading and autoloading it at the first call to it
+# will succeed.  In particular, if a function has been marked for autoloading
+# but is not available in $fpath, then this function will return False therefor.
+#
+# See users/21671 http://www.zsh.org/cgi-bin/mla/redirect?USERNUMBER=21671
+_zsh_highlight__function_callable_p() {
+  if _zsh_highlight__is_function_p "$1" &&
+     ! _zsh_highlight__function_is_autoload_stub_p "$1"
+  then
+    # Already fully loaded.
+    return 0 # true
+  else
+    # "$1" is either an autoload stub, or not a function at all.
+    #
+    # Use a subshell to avoid affecting the calling shell.
+    #
+    # We expect 'autoload +X' to return non-zero if it fails to fully load
+    # the function.
+    ( autoload -U +X -- "$1" 2>/dev/null )
+    return $?
+  fi
+}
+
 # -------------------------------------------------------------------------------------------------
 # Core highlighting update system
 # -------------------------------------------------------------------------------------------------
@@ -73,29 +119,93 @@ _zsh_highlight()
 {
   # Store the previous command return code to restore it whatever happens.
   local ret=$?
+  # Make it read-only.  Can't combine this with the previous line when POSIX_BUILTINS may be set.
+  typeset -r ret
+
+  # $region_highlight should be predefined, either by zle or by the test suite's mock (non-special) array.
+  (( ${+region_highlight} )) || {
+    echo >&2 'zsh-syntax-highlighting: error: $region_highlight is not defined'
+    echo >&2 'zsh-syntax-highlighting: (Check whether zsh-syntax-highlighting was installed according to the instructions.)'
+    return $ret
+  }
+
+  # Probe the memo= feature, once.
+  (( ${+zsh_highlight__memo_feature} )) || {
+    region_highlight+=( " 0 0 fg=red, memo=zsh-syntax-highlighting" )
+    case ${region_highlight[-1]} in
+      ("0 0 fg=red")
+        # zsh 5.8 or earlier
+        integer -gr zsh_highlight__memo_feature=0
+        ;;
+      ("0 0 fg=red memo=zsh-syntax-highlighting")
+        # zsh 5.9 or later
+        integer -gr zsh_highlight__memo_feature=1
+        ;;
+      (" 0 0 fg=red, memo=zsh-syntax-highlighting") ;&
+      (*)
+        # We can get here in two ways:
+        #
+        # 1. When not running as a widget.  In that case, $region_highlight is
+        # not a special variable (= one with custom getter/setter functions
+        # written in C) but an ordinary one, so the third case pattern matches
+        # and we fall through to this block.  (The test suite uses this codepath.)
+        #
+        # 2. When running under a future version of zsh that will have changed
+        # the serialization of $region_highlight elements from their underlying
+        # C structs, so that none of the previous case patterns will match.
+        #
+        # In either case, fall back to a version check.
+        #
+        # The memo= feature was added to zsh in commit zsh-5.8-172-gdd6e702ee.
+        # The version number at the time was 5.8.0.2-dev (see Config/version.mk).
+        # Therefore, on 5.8.0.3 and newer the memo= feature is available.
+        #
+        # On zsh version 5.8.0.2 between the aforementioned commit and the
+        # first Config/version.mk bump after it (which, at the time of writing,
+        # is yet to come), this condition will false negative.
+        if is-at-least 5.8.0.3; then
+          integer -gr zsh_highlight__memo_feature=1
+        else
+          integer -gr zsh_highlight__memo_feature=0
+        fi
+        ;;
+    esac
+    region_highlight[-1]=()
+  }
+
+  # Reset region_highlight to build it from scratch
+  if (( zsh_highlight__memo_feature )); then
+    region_highlight=( "${(@)region_highlight:#*memo=zsh-syntax-highlighting*}" )
+  else
+    # Legacy codepath.  Not very interoperable with other plugins (issue #418).
+    region_highlight=()
+  fi
 
   # Remove all highlighting in isearch, so that only the underlining done by zsh itself remains.
   # For details see FAQ entry 'Why does syntax highlighting not work while searching history?'.
   # This disables highlighting during isearch (for reasons explained in README.md) unless zsh is new enough
   # and doesn't have the pattern matching bug
   if [[ $WIDGET == zle-isearch-update ]] && { $zsh_highlight__pat_static_bug || ! (( $+ISEARCHMATCH_ACTIVE )) }; then
-    region_highlight=()
     return $ret
   fi
 
   # Before we 'emulate -L', save the user's options
   local -A zsyh_user_options
   if zmodload -e zsh/parameter; then
-    zsyh_user_options=("${(@kv)options}")
+    zsyh_user_options=("${(kv)options[@]}")
   else
     local canonical_options onoff option raw_options
     raw_options=(${(f)"$(emulate -R zsh; set -o)"})
     canonical_options=(${${${(M)raw_options:#*off}%% *}#no} ${${(M)raw_options:#*on}%% *})
-    for option in $canonical_options; do
+    for option in "${canonical_options[@]}"; do
       [[ -o $option ]]
-      # This variable cannot be eliminated c.f. workers/42101.
+      case $? in
-      onoff=${${=:-off on}[2-$?]}
+        (0) zsyh_user_options+=($option on);;
-      zsyh_user_options+=($option $onoff)
+        (1) zsyh_user_options+=($option off);;
+        (*) # Can't happen, surely?
+            echo "zsh-syntax-highlighting: warning: '[[ -o $option ]]' returned $?"
+            ;;
+      esac
     done
   fi
   typeset -r zsyh_user_options
@@ -111,10 +221,6 @@ _zsh_highlight()
   # Do not highlight if there are pending inputs (copy/paste).
   [[ $PENDING -gt 0 ]] && return $ret
 
-  # Reset region highlight to build it from scratch
-  typeset -ga region_highlight
-  region_highlight=();
-
   {
     local cache_place
     local -a region_highlight_copy
@@ -233,7 +339,7 @@ _zsh_highlight_apply_zle_highlight() {
   else
     start=$second end=$first
   fi
-  region_highlight+=("$start $end $region")
+  region_highlight+=("$start $end $region, memo=zsh-syntax-highlighting")
 }
 
 
@@ -273,7 +379,7 @@ _zsh_highlight_add_highlight()
   shift 2
   for highlight; do
     if (( $+ZSH_HIGHLIGHT_STYLES[$highlight] )); then
-      region_highlight+=("$start $end $ZSH_HIGHLIGHT_STYLES[$highlight]")
+      region_highlight+=("$start $end $ZSH_HIGHLIGHT_STYLES[$highlight], memo=zsh-syntax-highlighting")
       break
     fi
   done
@@ -287,76 +393,120 @@ _zsh_highlight_add_highlight()
 # $1 is name of widget to call
 _zsh_highlight_call_widget()
 {
-  builtin zle "$@" && 
+  builtin zle "$@" &&
   _zsh_highlight
 }
 
-# Rebind all ZLE widgets to make them invoke _zsh_highlights.
+# Decide whether to use the zle-line-pre-redraw codepath (colloquially known as
-_zsh_highlight_bind_widgets()
+# "feature/redrawhook", after the topic branch's name) or the legacy "bind all
-{
+# widgets" codepath.
-  setopt localoptions noksharrays
+#
-  typeset -F SECONDS
+# We use the new codepath under two conditions:
-  local prefix=orig-s$SECONDS-r$RANDOM # unique each time, in case we're sourced more than once
+#
-
+# 1. If it's available, which we check by testing for add-zle-hook-widget's availability.
-  # Load ZSH module zsh/zleparameter, needed to override user defined widgets.
+# 
-  zmodload zsh/zleparameter 2>/dev/null || {
+# 2. If zsh has the memo= feature, which is required for interoperability reasons.
-    print -r -- >&2 'zsh-syntax-highlighting: failed loading zsh/zleparameter.'
+#    See issues #579 and #735, and the issues referenced from them.
-    return 1
+#
+#    We check this with a plain version number check, since a functional check,
+#    as done by _zsh_highlight, can only be done from inside a widget
+#    function — a catch-22.
+#
+#    See _zsh_highlight for the magic version number.  (The use of 5.8.0.2
+#    rather than 5.8.0.3 as in the _zsh_highlight is deliberate.)
+if is-at-least 5.8.0.2 && _zsh_highlight__function_callable_p add-zle-hook-widget
+then
+  autoload -U add-zle-hook-widget
+  _zsh_highlight__zle-line-finish() {
+    # Reset $WIDGET since the 'main' highlighter depends on it.
+    #
+    # Since $WIDGET is declared by zle as read-only in this function's scope,
+    # a nested function is required in order to shadow its built-in value;
+    # see "User-defined widgets" in zshall.
+    () {
+      local -h -r WIDGET=zle-line-finish
+      _zsh_highlight
+    }
   }
+  _zsh_highlight__zle-line-pre-redraw() {
+    # Set $? to 0 for _zsh_highlight.  Without this, subsequent
+    # zle-line-pre-redraw hooks won't run, since add-zle-hook-widget happens to
+    # call us with $? == 1 in the common case.
+    true && _zsh_highlight "$@"
+  }
+  _zsh_highlight_bind_widgets(){}
+  if [[ -o zle ]]; then
+    add-zle-hook-widget zle-line-pre-redraw _zsh_highlight__zle-line-pre-redraw
+    add-zle-hook-widget zle-line-finish _zsh_highlight__zle-line-finish
+  fi
+else
+  # Rebind all ZLE widgets to make them invoke _zsh_highlights.
+  _zsh_highlight_bind_widgets()
+  {
+    setopt localoptions noksharrays
+    typeset -F SECONDS
+    local prefix=orig-s$SECONDS-r$RANDOM # unique each time, in case we're sourced more than once
 
-  # Override ZLE widgets to make them invoke _zsh_highlight.
+    # Load ZSH module zsh/zleparameter, needed to override user defined widgets.
-  local -U widgets_to_bind
+    zmodload zsh/zleparameter 2>/dev/null || {
-  widgets_to_bind=(${${(k)widgets}:#(.*|run-help|which-command|beep|set-local-history|yank|yank-pop)})
+      print -r -- >&2 'zsh-syntax-highlighting: failed loading zsh/zleparameter.'
+      return 1
+    }
 
-  # Always wrap special zle-line-finish widget. This is needed to decide if the
+    # Override ZLE widgets to make them invoke _zsh_highlight.
-  # current line ends and special highlighting logic needs to be applied.
+    local -U widgets_to_bind
-  # E.g. remove cursor imprint, don't highlight partial paths, ...
+    widgets_to_bind=(${${(k)widgets}:#(.*|run-help|which-command|beep|set-local-history|yank|yank-pop)})
-  widgets_to_bind+=(zle-line-finish)
 
-  # Always wrap special zle-isearch-update widget to be notified of updates in isearch.
+    # Always wrap special zle-line-finish widget. This is needed to decide if the
-  # This is needed because we need to disable highlighting in that case.
+    # current line ends and special highlighting logic needs to be applied.
-  widgets_to_bind+=(zle-isearch-update)
+    # E.g. remove cursor imprint, don't highlight partial paths, ...
+    widgets_to_bind+=(zle-line-finish)
 
-  local cur_widget
+    # Always wrap special zle-isearch-update widget to be notified of updates in isearch.
-  for cur_widget in $widgets_to_bind; do
+    # This is needed because we need to disable highlighting in that case.
-    case ${widgets[$cur_widget]:-""} in
+    widgets_to_bind+=(zle-isearch-update)
 
-      # Already rebound event: do nothing.
+    local cur_widget
-      user:_zsh_highlight_widget_*);;
+    for cur_widget in $widgets_to_bind; do
+      case ${widgets[$cur_widget]:-""} in
 
-      # The "eval"'s are required to make $cur_widget a closure: the value of the parameter at function
+        # Already rebound event: do nothing.
-      # definition time is used.
+        user:_zsh_highlight_widget_*);;
-      #
-      # We can't use ${0/_zsh_highlight_widget_} because these widgets are always invoked with
-      # NO_function_argzero, regardless of the option's setting here.
 
-      # User defined widget: override and rebind old one with prefix "orig-".
+        # The "eval"'s are required to make $cur_widget a closure: the value of the parameter at function
-      user:*) zle -N $prefix-$cur_widget ${widgets[$cur_widget]#*:}
+        # definition time is used.
-              eval "_zsh_highlight_widget_${(q)prefix}-${(q)cur_widget}() { _zsh_highlight_call_widget ${(q)prefix}-${(q)cur_widget} -- \"\$@\" }"
+        #
-              zle -N $cur_widget _zsh_highlight_widget_$prefix-$cur_widget;;
+        # We can't use ${0/_zsh_highlight_widget_} because these widgets are always invoked with
+        # NO_function_argzero, regardless of the option's setting here.
 
-      # Completion widget: override and rebind old one with prefix "orig-".
+        # User defined widget: override and rebind old one with prefix "orig-".
-      completion:*) zle -C $prefix-$cur_widget ${${(s.:.)widgets[$cur_widget]}[2,3]} 
+        user:*) zle -N $prefix-$cur_widget ${widgets[$cur_widget]#*:}
-                    eval "_zsh_highlight_widget_${(q)prefix}-${(q)cur_widget}() { _zsh_highlight_call_widget ${(q)prefix}-${(q)cur_widget} -- \"\$@\" }"
+                eval "_zsh_highlight_widget_${(q)prefix}-${(q)cur_widget}() { _zsh_highlight_call_widget ${(q)prefix}-${(q)cur_widget} -- \"\$@\" }"
-                    zle -N $cur_widget _zsh_highlight_widget_$prefix-$cur_widget;;
+                zle -N $cur_widget _zsh_highlight_widget_$prefix-$cur_widget;;
 
-      # Builtin widget: override and make it call the builtin ".widget".
+        # Completion widget: override and rebind old one with prefix "orig-".
-      builtin) eval "_zsh_highlight_widget_${(q)prefix}-${(q)cur_widget}() { _zsh_highlight_call_widget .${(q)cur_widget} -- \"\$@\" }"
+        completion:*) zle -C $prefix-$cur_widget ${${(s.:.)widgets[$cur_widget]}[2,3]}
-               zle -N $cur_widget _zsh_highlight_widget_$prefix-$cur_widget;;
+                      eval "_zsh_highlight_widget_${(q)prefix}-${(q)cur_widget}() { _zsh_highlight_call_widget ${(q)prefix}-${(q)cur_widget} -- \"\$@\" }"
+                      zle -N $cur_widget _zsh_highlight_widget_$prefix-$cur_widget;;
 
-      # Incomplete or nonexistent widget: Bind to z-sy-h directly.
+        # Builtin widget: override and make it call the builtin ".widget".
-      *) 
+        builtin) eval "_zsh_highlight_widget_${(q)prefix}-${(q)cur_widget}() { _zsh_highlight_call_widget .${(q)cur_widget} -- \"\$@\" }"
-         if [[ $cur_widget == zle-* ]] && (( ! ${+widgets[$cur_widget]} )); then
+                 zle -N $cur_widget _zsh_highlight_widget_$prefix-$cur_widget;;
-           _zsh_highlight_widget_${cur_widget}() { :; _zsh_highlight }
+
-           zle -N $cur_widget _zsh_highlight_widget_$cur_widget
+        # Incomplete or nonexistent widget: Bind to z-sy-h directly.
-         else
+        *)
-      # Default: unhandled case.
+           if [[ $cur_widget == zle-* ]] && (( ! ${+widgets[$cur_widget]} )); then
-           print -r -- >&2 "zsh-syntax-highlighting: unhandled ZLE widget ${(qq)cur_widget}"
+             _zsh_highlight_widget_${cur_widget}() { :; _zsh_highlight }
-           print -r -- >&2 "zsh-syntax-highlighting: (This is sometimes caused by doing \`bindkey <keys> ${(q-)cur_widget}\` without creating the ${(qq)cur_widget} widget with \`zle -N\` or \`zle -C\`.)"
+             zle -N $cur_widget _zsh_highlight_widget_$cur_widget
-         fi
+           else
-    esac
+        # Default: unhandled case.
-  done
+             print -r -- >&2 "zsh-syntax-highlighting: unhandled ZLE widget ${(qq)cur_widget}"
-}
+             print -r -- >&2 "zsh-syntax-highlighting: (This is sometimes caused by doing \`bindkey <keys> ${(q-)cur_widget}\` without creating the ${(qq)cur_widget} widget with \`zle -N\` or \`zle -C\`.)"
+           fi
+      esac
+    done
+  }
+fi
 
 # Load highlighters from directory.
 #
@@ -439,6 +589,20 @@ if (( $+X_ZSH_HIGHLIGHT_DIRS_BLACKLIST )); then
   unset X_ZSH_HIGHLIGHT_DIRS_BLACKLIST
 fi
 
+# Tell users of this branch (feature/redrawhook) to switch back to master.
+#
+# We don't want to nag them on every new shell, because that would be annoying.
+# We can't nag them once and never again, because we can't keep state.
+# Therefore, compromise: warn infrequently.
+#
+# Magic number: 63 is one less than a power of two.
+if (( (RANDOM & 63) == 42 )); then
+  print -r -- $'zsh-syntax-highlighting: You are using the feature/redrawhook branch.'
+  print -r -- $'zsh-syntax-highlighting: That branch has been merged to master and will not receive any further commits.'
+  print -r --  "zsh-syntax-highlighting: Consider switching ${(qq)0:h} back to the 'master' branch."
+fi
+
+
 # Restore the aliases we unned
 eval "$zsh_highlight__aliases"
 builtin unset zsh_highlight__aliases