Tag version 0.7.1.

See 1a55dc8fc2 (commitcomment-37476021)

.editorconfig

@@ -0,0 +1,9 @@
+# Top-most editorconfig file
+
+root = true
+
+[*]
+end_of_line = lf
+tab_width   = 2
+indent_size = 2
+indent_style = space

.version

@@ -1 +1 @@
-0.7.0-beta1
+0.7.1

COPYING.md

@@ -1,4 +1,4 @@
-Copyright (c) 2010-2018 zsh-syntax-highlighting contributors
+Copyright (c) 2010-2020 zsh-syntax-highlighting contributors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification, are permitted

INSTALL.md

@@ -6,7 +6,7 @@ How to install
 * Arch Linux: [community/zsh-syntax-highlighting][arch-package] / [AUR/zsh-syntax-highlighting-git][AUR-package]
 * Debian: `zsh-syntax-highlighting` package [in `stretch`][debian-package] (or in [OBS repository][obs-repository])
 * Fedora: [zsh-syntax-highlighting package][fedora-package-alt] in Fedora 24+ (or in [OBS repository][obs-repository])
-* FreeBSD: `pkg install zsh-syntax-highlighting` (port name: [`textproc/zsh-syntax-highlighting`][freebsd-port])
+* FreeBSD: `pkg install zsh-syntax-highlighting` (port name: [`shells/zsh-syntax-highlighting`][freebsd-port])
 * Gentoo: [mv overlay][gentoo-overlay]
 * Mac OS X / Homebrew: [brew install zsh-syntax-highlighting][brew-package]
 * Ubuntu: `zsh-syntax-highlighting` package [in Xenial][ubuntu-package] (or in [OBS repository][obs-repository])

changelog.md

@@ -1,6 +1,8 @@
-# Changes in version 0.7.0
+# Changes in version 0.7.1
 
-**Version 0.7.0 has not been released.  This changelog is a work in progress.**
+- Remove out-of-date information from the 0.7.0 changelog.
+
+# Changes in version 0.7.0
 
 This is a stable bugfix and feature release.  Major new features and changes include:
 
@@ -64,6 +66,14 @@ Selected bugfixes include:
   [#183]
 
 
+Known issues include:
+
+- A multiline alias that uses a simple command terminator (such as `;`, `|`, `&&`)
+  before a newline will incorrectly be highlighted as an error.  See issue #677
+  for examples and workarounds.
+  [#677]
+
+
 # Changes in version 0.6.0
 
 This is a stable release, featuring bugfixes and minor improvements.

docs/highlighters.md

@@ -104,7 +104,7 @@ To create your own `acme` highlighter:
         `_zsh_highlight_highlighter_acme_paint` respectively.
 
         These names are still supported for backwards compatibility;
-        however, support for them will be removed in a a future major or minor release (v0.x.0 or v1.0.0).
+        however, support for them will be removed in a future major or minor release (v0.x.0 or v1.0.0).
 
 * Activate your highlighter in `~/.zshrc`:
 

highlighters/main/main-highlighter.zsh

@@ -1,5 +1,5 @@
 # -------------------------------------------------------------------------------------------------
-# Copyright (c) 2010-2018 zsh-syntax-highlighting contributors
+# Copyright (c) 2010-2020 zsh-syntax-highlighting contributors
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification, are permitted
@@ -77,6 +77,16 @@ _zsh_highlight_main_add_region_highlight() {
     [[ $1 == unknown-token ]] && alias_style=unknown-token
     return
   fi
+  if (( in_param )); then
+    if [[ $1 == unknown-token ]]; then
+      param_style=unknown-token
+    fi
+    if [[ -n $param_style ]]; then
+      return
+    fi
+    param_style=$1
+    return
+  fi
 
   # The calculation was relative to $buf but region_highlight is relative to $BUFFER.
   (( start += buf_offset ))
@@ -188,7 +198,7 @@ _zsh_highlight_main__type() {
     elif {  [[ $1 != */* ]] || is-at-least 5.3 } &&
          # Add a subshell to avoid a zsh upstream bug; see issue #606.
          # ### Remove the subshell when we stop supporting zsh 5.7.1 (I assume 5.8 will have the bugfix).
-         ! (builtin type -w -- $1) >/dev/null 2>&1; then
+         ! (builtin type -w -- "$1") >/dev/null 2>&1; then
       REPLY=none
     fi
   fi
@@ -203,7 +213,7 @@ _zsh_highlight_main__type() {
     # starts with an arithmetic expression [«((…))» as the first thing inside
     # «$(…)»], which is area that has had some parsing bugs before 5.6
     # (approximately).
-    REPLY="${$(:; (( aliases_allowed )) || unalias -- $1 2>/dev/null; LC_ALL=C builtin type -w -- $1 2>/dev/null)##*: }"
+    REPLY="${$(:; (( aliases_allowed )) || unalias -- "$1" 2>/dev/null; LC_ALL=C builtin type -w -- "$1" 2>/dev/null)##*: }"
     if [[ $REPLY == 'alias' ]]; then
       may_cache=0
     fi
@@ -315,13 +325,27 @@ _zsh_highlight_highlighter_main_paint()
     'doas' aCu:Lns # as of OpenBSD's doas(1) dated September 4, 2016
     'nice' n: # as of current POSIX spec
     'pkexec' '' # doesn't take short options; immune to #121 because it's usually not passed --option flags
-    'sudo' Cgprtu:AEHKPSVbhiklnsv # as of sudo 1.8.21p2
+    # Argumentless flags that can't be followed by a command: -e -h -K -k -V -v
+    'sudo' Cgprtu:AEHPSbilns # as of sudo 1.8.21p2
     'stdbuf' ioe:
     'eatmydata' ''
     'catchsegv' ''
     'nohup' ''
     'setsid' :wc
+    # As of OpenSSH 8.1p1; -k is deliberately left out since it may not be followed by a command
+    'ssh-agent' aEPt:csDd
+    # suckless-tools v44
+    # Argumentless flags that can't be followed by a command: -v
+    'tabbed' gnprtTuU:cdfhs
+
+    # moreutils 0.62-1
+    'chronic' :ev
+    'ifne' :n
+
   )
+  # Commands that would need to skip one positional argument:
+  #    flock
+  #    ssh
 
   if [[ $zsyh_user_options[ignorebraces] == on || ${zsyh_user_options[ignoreclosebraces]:-off} == on ]]; then
     local right_brace_is_recognised_everywhere=false
@@ -392,13 +416,13 @@ _zsh_highlight_main_highlighter_highlight_list()
   # alias_style is the style to apply to an alias once in_alias=0
   #     Usually 'alias' but set to 'unknown-token' if any word expanded from
   #     the alias would be highlighted as unknown-token
-  # last_alias is the last alias arg (lhs) expanded (if in an alias).
-  #     This allows for expanding alias ls='ls -l' while avoiding loops.
-  local alias_style arg buf=$4 highlight_glob=true last_alias style
+  # param_style is analogous for parameter expansions
+  local alias_style param_style arg buf=$4 highlight_glob=true style
   local in_array_assignment=false # true between 'a=(' and the matching ')'
   # in_alias is equal to the number of shifts needed until arg=args[1] pops an
   #     arg from BUFFER and not added by an alias.
-  integer in_alias=0 len=$#buf
+  # in_param is analogous for parameter expansions
+  integer in_alias=0 in_param=0 len=$#buf
   local -a match mbegin mend list_highlights
   # seen_alias is a map of aliases already seen to avoid loops like alias a=b b=a
   local -A seen_alias
@@ -434,12 +458,12 @@ _zsh_highlight_main_highlighter_highlight_list()
   #
   # When the kind of a word is not yet known, $this_word / $next_word may contain
   # multiple states.  For example, after "sudo -i", the next word may be either
-  # another --flag or a command name, hence the state would include both :start:
-  # and :sudo_opt:.
+  # another --flag or a command name, hence the state would include both ':start:'
+  # and ':sudo_opt:'.
   #
   # The tokens are always added with both leading and trailing colons to serve as
-  # word delimiters (an improvised array); [[ $x == *:foo:* ]] and x=${x//:foo:/}
-  # will DTRT regardless of how many elements or repetitions $x has..
+  # word delimiters (an improvised array); [[ $x == *':foo:'* ]] and x=${x//:foo:/}
+  # will DTRT regardless of how many elements or repetitions $x has.
   #
   # Handling of redirections: upon seeing a redirection token, we must stall
   # the current state --- that is, the value of $this_word --- for two iterations
@@ -472,11 +496,19 @@ _zsh_highlight_main_highlighter_highlight_list()
     if (( in_alias )); then
       (( in_alias-- ))
       if (( in_alias == 0 )); then
-        last_alias= seen_alias=()
+        seen_alias=()
         # start_pos and end_pos are of the alias (previous $arg) here
         _zsh_highlight_main_add_region_highlight $start_pos $end_pos $alias_style
       fi
     fi
+    if (( in_param )); then
+      (( in_param-- ))
+      if (( in_param == 0 )); then
+        # start_pos and end_pos are of the '$foo' word (previous $arg) here
+        _zsh_highlight_main_add_region_highlight $start_pos $end_pos $param_style
+        param_style=""
+      fi
+    fi
 
     # Initialize this_word and next_word.
     if (( in_redirection == 0 )); then
@@ -501,7 +533,7 @@ _zsh_highlight_main_highlighter_highlight_list()
       fi
     fi
 
-    if (( in_alias == 0 )); then
+    if (( in_alias == 0 && in_param == 0 )); then
       # Compute the new $start_pos and $end_pos, skipping over whitespace in $buf.
       [[ "$proc_buf" = (#b)(#s)(([ $'\t']|\\$'\n')#)* ]]
       # The first, outer parenthesis
@@ -544,20 +576,19 @@ _zsh_highlight_main_highlighter_highlight_list()
       continue
     fi
 
-    if [[ $this_word == *:start:* ]] && ! (( in_redirection )); then
+    if [[ $this_word == *':start:'* ]] && ! (( in_redirection )); then
       # Expand aliases.
-      _zsh_highlight_main__type "$arg"
+      # An alias is ineligible for expansion while it's being expanded (see #652/#653).
+      _zsh_highlight_main__type "$arg" "$(( ! ${+seen_alias[$arg]} ))"
       local res="$REPLY"
-      if [[ $res == "alias" ]] && [[ $last_alias != $arg ]]; then
-        # Avoid looping forever on alias a=b b=c c=b, but allow alias foo='foo bar'
-        # Also mark insane aliases as unknown-token (cf. #263).
-        if (( $+seen_alias[$arg] )) || [[ $arg == ?*=* ]]; then
+      if [[ $res == "alias" ]]; then
+        # Mark insane aliases as unknown-token (cf. #263).
+        if [[ $arg == ?*=* ]]; then
           (( in_alias == 0 )) && in_alias=1
           _zsh_highlight_main_add_region_highlight $start_pos $end_pos unknown-token
           continue
         fi
         seen_alias[$arg]=1
-        last_alias=$arg
         _zsh_highlight_main__resolve_alias $arg
         local -a alias_args
         # Elision is desired in case alias x=''
@@ -580,8 +611,7 @@ _zsh_highlight_main_highlighter_highlight_list()
         continue
       else
         _zsh_highlight_main_highlighter_expand_path $arg
-        arg=$REPLY
-        _zsh_highlight_main__type "$arg" 0
+        _zsh_highlight_main__type "$REPLY" 0
         res="$REPLY"
       fi
     fi
@@ -616,6 +646,7 @@ _zsh_highlight_main_highlighter_highlight_list()
       local -a match mbegin mend
       local MATCH; integer MBEGIN MEND
       local parameter_name
+      local -a words
       if [[ $arg[1] == '$' ]] && [[ ${arg[2]} == '{' ]] && [[ ${arg[-1]} == '}' ]]; then
         parameter_name=${${arg:2}%?}
       elif [[ $arg[1] == '$' ]]; then
@@ -628,14 +659,16 @@ _zsh_highlight_main_highlighter_highlight_list()
         # Set $arg.
         case ${(tP)MATCH} in
           (*array*|*assoc*)
-            local -a words; words=( ${(P)MATCH} )
-            arg=${words[1]}
+            words=( ${(P)MATCH} )
             ;;
           (*)
             # scalar, presumably
-            arg=${(P)MATCH}
+            words=( ${(P)MATCH} )
             ;;
         esac
+        (( in_param = 1 + $#words ))
+        args=( $words $args )
+        arg=$args[1]
         _zsh_highlight_main__type "$arg" 0
         res=$REPLY
       fi
@@ -660,16 +693,20 @@ _zsh_highlight_main_highlighter_highlight_list()
         elif [[ -n $flags_sans_argument ]] &&
              [[ $arg == '-'[$flags_sans_argument]# ]]; then
           # Flag that requires no argument
-          this_word=:sudo_opt:
+          this_word=':sudo_opt:'
           next_word+=':start:'
           next_word+=':sudo_opt:'
         elif [[ $arg == '-'* ]]; then
-          # Unknown flag
-          this_word=:sudo_opt:
+          # Unknown flag.  We don't know whether it takes an argument or not,
+          # so modify $next_word as we do for flags that require no argument.
+          # With that behaviour, if the flag in fact takes no argument we'll
+          # highlight the inner command word correctly, and if it does take an
+          # argument we'll highlight the command word correctly if the argument
+          # was given in the same shell word as the flag (as in '-uphy1729' or
+          # '--user=phy1729' without spaces).
+          this_word=':sudo_opt:'
           next_word+=':start:'
           next_word+=':sudo_opt:'
-          _zsh_highlight_main_add_region_highlight $start_pos $end_pos unknown-token
-          continue
         else
           # Not an option flag; nothing to do.  (If the command line is
           # syntactically valid, ${this_word//:sudo_opt:/} should be
@@ -803,7 +840,7 @@ _zsh_highlight_main_highlighter_highlight_list()
         function)       style=function;;
         command)        style=command;;
         hashed)         style=hashed-command;;
-        none)           if _zsh_highlight_main_highlighter_check_assign; then
+        none)           if (( ! in_param )) && _zsh_highlight_main_highlighter_check_assign; then
                           _zsh_highlight_main_add_region_highlight $start_pos $end_pos assign
                           local i=$(( arg[(i)=] + 1 ))
                           if [[ $arg[i] == '(' ]]; then
@@ -824,11 +861,14 @@ _zsh_highlight_main_highlighter_highlight_list()
                             fi
                           fi
                           continue
-                        elif [[ $arg[0,1] = $histchars[0,1] ]] && (( $#arg[0,2] == 2 )); then
+                        elif (( ! in_param )) &&
+                             [[ $arg[0,1] = $histchars[0,1] ]] && (( $#arg[0,2] == 2 )); then
                           style=history-expansion
-                        elif [[ $arg[0,1] == $histchars[2,2] ]]; then
+                        elif (( ! in_param )) &&
+                             [[ $arg[0,1] == $histchars[2,2] ]]; then
                           style=history-expansion
-                        elif [[ $arg[1,2] == '((' ]]; then
+                        elif (( ! in_param )) &&
+                             [[ $arg[1,2] == '((' ]]; then
                           # Arithmetic evaluation.
                           #
                           # Note: prior to zsh-5.1.1-52-g4bed2cf (workers/36669), the ${(z)...}
@@ -843,14 +883,17 @@ _zsh_highlight_main_highlighter_highlight_list()
                             _zsh_highlight_main_add_region_highlight $((end_pos - 2)) $end_pos reserved-word
                           fi
                           continue
-                        elif [[ $arg == '()' ]]; then
+                        elif (( ! in_param )) &&
+                             [[ $arg == '()' ]]; then
                           # anonymous function
                           style=reserved-word
-                        elif [[ $arg == $'\x28' ]]; then
+                        elif (( ! in_param )) &&
+                             [[ $arg == $'\x28' ]]; then
                           # subshell
                           style=reserved-word
                           braces_stack='R'"$braces_stack"
-                        elif [[ $arg == $'\x29' ]]; then
+                        elif (( ! in_param )) &&
+                             [[ $arg == $'\x29' ]]; then
                           # end of subshell or command substitution
                           if _zsh_highlight_main__stack_pop 'S'; then
                             REPLY=$start_pos
@@ -932,6 +975,7 @@ _zsh_highlight_main_highlighter_highlight_list()
     _zsh_highlight_main_add_region_highlight $start_pos $end_pos $style
   done
   (( in_alias == 1 )) && in_alias=0 _zsh_highlight_main_add_region_highlight $start_pos $end_pos $alias_style
+  (( in_param == 1 )) && in_param=0 _zsh_highlight_main_add_region_highlight $start_pos $end_pos $param_style
   [[ "$proc_buf" = (#b)(#s)(([[:space:]]|\\$'\n')#) ]]
   REPLY=$(( end_pos + ${#match[1]} - 1 ))
   reply=($list_highlights)

highlighters/main/test-data/abspath-in-command-position1.zsh

@@ -0,0 +1,35 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$'/'
+
+expected_region_highlight=(
+  '1 1 path' # /
+)

highlighters/main/test-data/abspath-in-command-position2.zsh

@@ -0,0 +1,35 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$'/bi'
+
+expected_region_highlight=(
+  '1 3 path_prefix' # /bi
+)

highlighters/main/test-data/abspath-in-command-position3.zsh

@@ -0,0 +1,35 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$'/bin'
+
+expected_region_highlight=(
+  '1 4 path' # /bin
+)

highlighters/main/test-data/abspath-in-command-position4.zsh

@@ -0,0 +1,35 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$'/bin/s'
+
+expected_region_highlight=(
+  '1 6 path_prefix' # /bin/s
+)

highlighters/main/test-data/abspath-in-command-position5.zsh

@@ -0,0 +1,35 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$'/bin/sh'
+
+expected_region_highlight=(
+  '1 7 command' # /bin/sh
+)

highlighters/main/test-data/alias-eponymous2.zsh

@@ -32,5 +32,5 @@ alias ls=tmp tmp='command ls'
 BUFFER='ls'
 
 expected_region_highlight=(
-  "1 2 alias 'issue #652'" # ls
+  "1 2 alias" # ls
 )

highlighters/main/test-data/alias-parameter.zsh

@@ -0,0 +1,38 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+alias '$foo'='echo alias'
+local foo; foo=(echo param)
+
+BUFFER='$foo'
+
+expected_region_highlight=(
+  '1 4 alias' # $foo
+)

highlighters/main/test-data/alias-precommand-option-argument1.zsh

@@ -28,6 +28,7 @@
 # vim: ft=zsh sw=2 ts=2 et
 # -------------------------------------------------------------------------------------------------
 
+# See also param-precommand-option-argument1.zsh
 alias sudo_u='sudo -u'
 sudo(){}
 

highlighters/main/test-data/alias-precommand-option-argument3.zsh

@@ -28,6 +28,7 @@
 # vim: ft=zsh sw=2 ts=2 et
 # -------------------------------------------------------------------------------------------------
 
+# See also param-precommand-option-argument3.zsh
 alias sudo_u='sudo -u'
 sudo(){}
 

highlighters/main/test-data/alias-to-dir.zsh

@@ -32,5 +32,5 @@ alias x=/
 BUFFER=$'x'
 
 expected_region_highlight=(
-  '1 1 unknown-token "issue #202"' # x (/)
+  '1 1 unknown-token "issue #668"' # x (/)
 )

highlighters/main/test-data/assign-value-quote1.zsh

@@ -0,0 +1,37 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$'s="foo\'bar"'
+
+expected_region_highlight=(
+  '1 11 assign' # s="foo'bar"
+  '3 11 default' # "foo'bar"
+  '3 11 double-quoted-argument' # "foo'bar"
+)

highlighters/main/test-data/assign-value-quote2.zsh

@@ -0,0 +1,37 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$'s="foo \'\' bar"'
+
+expected_region_highlight=(
+  '1 14 assign' # s="foo '' bar"
+  '3 14 default' # "foo '' bar"
+  '3 14 double-quoted-argument' # "foo '' bar"
+)

highlighters/main/test-data/assignment-quoted.zsh

@@ -0,0 +1,37 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$'1="foo"'
+
+expected_region_highlight=(
+  '1 7 assign' # 1="foo"
+  '3 7 default' # "foo"
+  '3 7 double-quoted-argument' # "foo"
+)

highlighters/main/test-data/cmdpos-elision-partial.zsh

@@ -0,0 +1,44 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+# Test elision of some, but not all of the words
+# See issue #667 for the case of eliding all words
+local -a x; x=(sudo "")
+
+sudo(){}
+BUFFER=$'$x -u phy1729 ls'
+
+expected_region_highlight=(
+  '1 2 precommand' # $x
+  # The "" is elided.  If it weren't elided, the «ls» would be highlighted as an ordinary argument.
+  '4 5 single-hyphen-option' # -u
+  '7 13 default' # phy1729
+  '15 16 command' # ls
+)

highlighters/main/test-data/null-exec.zsh

@@ -0,0 +1,38 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$'exec >/dev/null;'
+
+expected_region_highlight=(
+  '1 4 precommand' # exec
+  '6 6 redirection' # >
+  '7 15 path' # /dev/null
+  '16 16 commandseparator "issue #676"' # ;
+)

highlighters/main/test-data/param-precommand-option-argument1.zsh

@@ -0,0 +1,42 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+# See also alias-precommand-option-argument1.zsh
+local -a sudo_u; sudo_u=(sudo -u)
+sudo(){}
+
+BUFFER='$sudo_u phy1729 echo foo'
+
+expected_region_highlight=(
+  '1 7 precommand' # $sudo_u
+  '9 15 default' # phy1729
+  '18 20 command "issue #540"' # echo (not builtin)
+  '22 24 default' # foo
+)

highlighters/main/test-data/param-precommand-option-argument3.zsh

@@ -0,0 +1,42 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+# See also alias-precommand-option-argument3.zsh
+local -a sudo_u; sudo_u=(sudo -u)
+sudo(){}
+
+BUFFER='$sudo_u phy1729 ls foo'
+
+expected_region_highlight=(
+  '1 7 precommand' # sudo_u
+  '9 15 default' # phy1729
+  '17 18 command' # ls
+  '20 22 default' # foo
+)

highlighters/main/test-data/parameter-expansion-untokenized1.zsh

@@ -0,0 +1,38 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+local x="()"
+
+BUFFER=$'$x ls'
+
+expected_region_highlight=(
+  '1 2 unknown-token' # $x
+  '4 5 command' # ls
+)

highlighters/main/test-data/parameter-expansion-untokenized2.zsh

@@ -0,0 +1,38 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+local x="^foo^bar"
+
+BUFFER=$'$x ls'
+
+expected_region_highlight=(
+  '1 2 unknown-token' # $x
+  '4 5 default' # ls
+)

highlighters/main/test-data/parameter-value-contains-command-position1.zsh

@@ -0,0 +1,38 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+local foobar='x=$(ls)'
+
+BUFFER=$'$foobar'
+
+expected_region_highlight=(
+  # Used to highlight the "ba" as 'command' because the 'ls' showed through; issues #670 and #674
+  '1 7 unknown-token' # $foobar (not an assignment)
+)

highlighters/main/test-data/parameter-value-contains-command-position2.zsh

@@ -0,0 +1,38 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+local y='x=$(ls)'
+
+BUFFER=$'$y'
+
+expected_region_highlight=(
+  # Used to trigger a "BUG" message on stderr - issues #670 and #674
+  '1 2 unknown-token' # $y (not an assignment)
+)

highlighters/main/test-data/precommand-killing1.zsh

@@ -0,0 +1,39 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+hash sudo=false
+
+BUFFER='sudo -e /etc/passwd'
+
+expected_region_highlight=(
+  '1 4 precommand' # sudo
+  '6 7 single-hyphen-option' # -e
+  '9 19 path' # /etc/passwd
+)

highlighters/main/test-data/precommand-killing2.zsh

@@ -0,0 +1,39 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+hash sudo=false
+
+BUFFER='sudo -e /does/not/exist'
+
+expected_region_highlight=(
+  '1 4 precommand' # sudo
+  '6 7 single-hyphen-option' # -e
+  '9 23 normal "issue #678"' # /does/not/exist
+)

highlighters/main/test-data/precommand-unknown-option.zsh

@@ -38,6 +38,6 @@ expected_region_highlight=(
   '10 11 command' # ls
   '12 12 commandseparator' # ;
   '14 17 precommand' # sudo
-  '19 20 unknown-token' # -x
+  '19 20 single-hyphen-option' # -x
   '22 23 command' # ls
 )

highlighters/main/test-data/quoted-redirection-in-command-word.zsh

@@ -0,0 +1,37 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=$'">" foo ls'
+
+expected_region_highlight=(
+  '1 3 unknown-token' # ">" - not "redirection"
+  '5 7 default' # foo
+  '9 10 default' # ls - not "command"
+)

highlighters/main/test-data/sudo-longopt.zsh

@@ -0,0 +1,38 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+hash sudo='false'
+BUFFER='sudo --askpass ls'
+
+expected_region_highlight=(
+  '1 4 precommand' # sudo
+  '6 14 double-hyphen-option' # --askpass
+  '16 17 command' # ls (we don't know whether --askpass takes an argument)
+)

release.md

@@ -5,11 +5,13 @@
 - Check open issues and outstanding pull requests
 - Confirm `make test` passes
   - check with multiple zsh versions
+    (easiest to check travis: https://travis-ci.org/zsh-users/zsh-syntax-highlighting/)
 - Update changelog.md
   `tig --abbrev=12  --abbrev-commit 0.4.1..upstream/master`
-- Remove `-dev` suffix from `./.version`;
+- Make sure there are no local commits and that `git status` is clean;
+  Remove `-dev` suffix from `./.version`;
   Commit that using `git commit -m "Tag version $(<.version)." .version`;
-  Tag it using `git tag -s -m "Tag version $(<.version)"`;
+  Tag it using `git tag -s -m "Tag version $(<.version)" $(<.version)`;
   Increment `./.version` and restore the `-dev` suffix;
   Commit that using `git commit -C b5c30ae52638e81a38fe5329081c5613d7bd6ca5 .version`.
 - Push with `git push && git push --tags`

tests/edit-failed-tests

@@ -0,0 +1,40 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+type perl sponge >/dev/null || { print -ru2 -- "$0: This script requires perl(1) and sponge(1) [from moreutils]"; exit 1; }
+
+local editor=( "${(@Q)${(z)${VISUAL:-${EDITOR:-vi}}}}" )
+() { 
+  > "$2" perl -nE '$highlighter = $1 if /^Running test (\S*)/; say "highlighters/${highlighter}/test-data/$1.zsh" if /^# (\S*)/' "$1"
+  >>"$2" echo ""
+  >>"$2" cat <"$1"
+  "${editor[@]}" -- "$2"
+} =(${MAKE:-make} quiet-test) =(:)
+# TODO: tee(1) the quiet-test output to /dev/tty as it's happening, with colors.

tests/generate.zsh

@@ -32,16 +32,17 @@ emulate -LR zsh
 setopt localoptions extendedglob
 
 # Argument parsing.
-if (( $# != 3 )) || [[ $1 == -* ]]; then
-  print -r -- >&2 "$0: usage: $0 BUFFER HIGHLIGHTER BASENAME"
+if (( $# * $# - 7 * $# + 12 )) || [[ $1 == -* ]]; then
+  print -r -- >&2 "$0: usage: $0 BUFFER HIGHLIGHTER BASENAME [PREAMBLE]"
   print -r -- >&2 ""
   print -r -- >&2 "Generate highlighters/HIGHLIGHTER/test-data/BASENAME.zsh based on the"
-  print -r -- >&2 "current highlighting of BUFFER."
+  print -r -- >&2 "current highlighting of BUFFER, using the setup code PREAMBLE."
   exit 1
 fi
 buffer=$1
 ZSH_HIGHLIGHT_HIGHLIGHTERS=( $2 )
 fname=${0:A:h:h}/highlighters/$2/test-data/${3%.zsh}.zsh
+preamble=${4:-""}
 
 # Load the main script.
 . ${0:A:h:h}/zsh-syntax-highlighting.zsh
@@ -61,9 +62,14 @@ fi
 <$0 sed -n -e '1,/^$/p' | sed -e "s/2[0-9][0-9][0-9]/${year}/" > $fname
 # Assumes stdout is line-buffered
 git add -- $fname
+exec > >(tee -a $fname)
+
+# Preamble
+if [[ -n $preamble ]]; then
+  print -rl -- "$preamble" ""
+fi
 
 # Buffer
-exec > >(tee -a $fname)
 print -n 'BUFFER='
 if [[ $buffer != (#s)[$'\t -~']#(#e) ]]; then
   print -r -- ${(qqqq)buffer}
@@ -82,8 +88,17 @@ print 'expected_region_highlight=('
   PREBUFFER=""
   BUFFER="$buffer"
   region_highlight=()
-  # TODO: use run_test() from tests/test-highlighting.zsh (to get a tempdir)
-  _zsh_highlight
+  eval $(
+    exec 3>&1 >/dev/null
+    typeset -r __tests_tmpdir="$(mktemp -d)"
+    {
+      # Use a subshell to ensure $__tests_tmpdir, which is to be rm -rf'd, won't be modified.
+      (cd -- "$__tests_tmpdir" && eval $preamble && _zsh_highlight && typeset -p region_highlight >&3)
+      : # workaround zsh bug workers/45305 with respect to the $(…) subshell we're in
+    } always {
+      rm -rf -- ${__tests_tmpdir}
+    }
+  )
 
   for ((i=1; i<=${#region_highlight}; i++)); do
     local -a highlight_zone; highlight_zone=( ${(z)region_highlight[$i]} )

tests/tap-filter

@@ -40,6 +40,8 @@ use strict;
 
 undef $/; # slurp mode
 print for
-  grep { /^ok.*# TODO/m or /^not ok(?!.*# TODO)/m }
-    split /^(?=#)/m,
+  grep { /^ok.*# TODO/m or /^not ok(?!.*# TODO)/m or /^Bail out!/m }
+    # Split on plan lines and remove them from the output.  (To keep them,
+    # use the lookahead syntax, «(?=…)», to make the match zero-length.)
+    split /^\d+\.\.\d+$/m,
       <STDIN>;

tests/test-highlighting.zsh

@@ -93,9 +93,13 @@ ZSH_HIGHLIGHT_HIGHLIGHTERS=($1)
 
 # In zsh<5.3, 'typeset -p arrayvar' emits two lines, so we use this wrapper instead.
 typeset_p() {
-	for 1 ; do
-		print -r -- "$1=( ${(@q-P)1} )"
-	done
+  for 1 ; do
+    if [[ ${(tP)1} == *array* ]]; then
+      print -r -- "$1=( ${(@q-P)1} )"
+    else
+      print -r -- "$1=${(q-P)1}"
+    fi
+  done
 }
 
 # Escape # as ♯ and newline as ↵ they are illegal in the 'description' part of TAP output
@@ -113,12 +117,11 @@ run_test_internal() {
   local srcdir="$PWD"
   builtin cd -q -- "$tests_tempdir" || { echo >&2 "Bail out! On ${(qq)1}: cd failed: $?"; return 1 }
 
-  echo "# ${1:t:r}"
-
   # Load the data and prepare checking it.
   local BUFFER CURSOR MARK PENDING PREBUFFER REGION_ACTIVE WIDGET REPLY skip_test unsorted=0
   local expected_mismatch
   local -a expected_region_highlight region_highlight
+
   . "$srcdir"/"$1"
 
   (( $#skip_test )) && { print -r -- "1..0 # SKIP $skip_test"; return; }
@@ -139,7 +142,12 @@ run_test_internal() {
     expected_region_highlight=("${(@n)expected_region_highlight}")
   fi
 
+  # Print the plan line, and some comments for human readers
   echo "1..$(( $#expected_region_highlight + 1))"
+  echo "## ${1:t:r}"
+  [[ -n $PREBUFFER ]] && printf '# %s\n' "$(typeset_p PREBUFFER)"
+  [[ -n $BUFFER ]] && printf '# %s\n' "$(typeset_p BUFFER)"
+
   local i
   for ((i=1; i<=$#expected_region_highlight; i++)); do
     local -a expected_highlight_zone; expected_highlight_zone=( ${(z)expected_region_highlight[i]} )
@@ -202,7 +210,7 @@ run_test() {
       local ret=$pipestatus[1] stderr=$pipestatus[2]
       if (( ! stderr )); then
         # stdout will become stderr
-	echo "Bail out! On ${(qq)1}: output on stderr"; return 1
+        echo "Bail out! On ${(qq)1}: output on stderr"; return 1
       else
         return $ret
       fi

zsh-syntax-highlighting.zsh

@@ -161,12 +161,12 @@ _zsh_highlight()
       (( REGION_ACTIVE )) || return
       integer min max
       if (( MARK > CURSOR )) ; then
-	min=$CURSOR max=$MARK
+        min=$CURSOR max=$MARK
       else
-	min=$MARK max=$CURSOR
+        min=$MARK max=$CURSOR
       fi
       if (( REGION_ACTIVE == 1 )); then
-	[[ $KEYMAP = vicmd ]] && (( max++ ))
+        [[ $KEYMAP = vicmd ]] && (( max++ ))
       elif (( REGION_ACTIVE == 2 )); then
         local needle=$'\n'
         # CURSOR and MARK are 0 indexed between letters like region_highlight