Tag version 0.4.1

(cherry picked from commit 683f4837a0)

.gitignore

@@ -1 +1,3 @@
 *.zwc*
+.pc/
+docs/all.md

.version

@@ -1 +1 @@
-0.3.0
+0.4.1

INSTALL.md

@@ -0,0 +1,70 @@
+How to install
+--------------
+
+### Using packages
+
+* Arch Linux: [community/zsh-syntax-highlighting][arch-package] / [AUR/zsh-syntax-highlighting-git][AUR-package]
+* Gentoo: [mv overlay][gentoo-overlay]
+* Mac OS X / Homebrew: [brew install zsh-syntax-highlighting][brew-package]
+
+[arch-package]: https://www.archlinux.org/packages/zsh-syntax-highlighting
+[AUR-package]: https://aur.archlinux.org/packages/zsh-syntax-highlighting-git
+[gentoo-overlay]: http://gpo.zugaina.org/app-shells/zsh-syntax-highlighting
+[brew-package]: https://github.com/Homebrew/homebrew/blob/master/Library/Formula/zsh-syntax-highlighting.rb
+
+
+### In your ~/.zshrc
+
+Simply clone this repository and source the script:
+
+        git clone git://github.com/zsh-users/zsh-syntax-highlighting.git
+        echo "source ${(q-)PWD}/zsh-syntax-highlighting/zsh-syntax-highlighting.zsh" >> ~/.zshrc
+        source ~/.zshrc
+
+  If `git` is not installed, download and extract a snapshot of the latest
+  development tree from:
+
+        https://github.com/zsh-users/zsh-syntax-highlighting/archive/master.tar.gz
+
+  Note the `source` command must be **at the end** of `~/.zshrc`.
+
+
+### With oh-my-zsh
+
+Oh-my-zsh is a zsh configuration framework.  It lives at
+<http://github.com/robbyrussell/oh-my-zsh>.
+
+To install zsh-syntax-highlighting under oh-my-zsh:
+
+1. Clone this repository in oh-my-zsh's plugins directory:
+
+        git clone git://github.com/zsh-users/zsh-syntax-highlighting.git ~/.oh-my-zsh/custom/plugins/zsh-syntax-highlighting
+
+2. Activate the plugin in `~/.zshrc`:
+
+        plugins=( [plugins...] zsh-syntax-highlighting)
+
+3. Source `~/.zshrc`  to take changes into account:
+
+        source ~/.zshrc
+
+Note that `zsh-syntax-highlighting` must be the last plugin sourced,
+so make it the last element of the `$plugins` array.
+
+
+### System-wide installation
+
+Either of the above methods is suitable for a single-user installation,
+which requires no special privileges.  If, however, you desire to install
+zsh-syntax-highlighting system-wide, you may do so by running
+
+    make install
+
+and directing your users to add
+
+    source /usr/local/share/zsh-syntax-highlighting/zsh-syntax-highlighting.zsh
+
+to their `.zshrc`s.
+
+
+

Makefile

@@ -2,30 +2,58 @@ NAME=zsh-syntax-highlighting
 
 INSTALL?=install -c
 PREFIX?=/usr/local
-SHARE_DIR=$(DESTDIR)$(PREFIX)/share/$(NAME)
+SHARE_DIR?=$(DESTDIR)$(PREFIX)/share/$(NAME)
+DOC_DIR?=$(DESTDIR)$(PREFIX)/share/doc/$(NAME)
+ZSH?=zsh # zsh binary to run tests with
 
-# Have the default target do nothing.
 all:
-	@ :
+	cd docs && \
+	cp highlighters.md all.md && \
+	printf '\n\nIndividual highlighters documentation\n=====================================' >> all.md && \
+	for doc in highlighters/*.md; do printf '\n\n'; cat "$$doc"; done >> all.md
 
-install:
+install: all
 	$(INSTALL) -d $(SHARE_DIR)
-	cp -r .version zsh-syntax-highlighting.zsh highlighters $(SHARE_DIR)
+	$(INSTALL) -d $(DOC_DIR)
+	cp .version zsh-syntax-highlighting.zsh $(SHARE_DIR)
+	cp COPYING.md README.md changelog.md $(DOC_DIR)
 	if [ x"true" = x"`git rev-parse --is-inside-work-tree 2>/dev/null`" ]; then \
 		git rev-parse HEAD; \
 	else \
 		cat .revision-hash; \
 	fi > $(SHARE_DIR)/.revision-hash
+	:
+# The [ -e ] check below is to because sh evaluates this with (the moral
+# equivalent of) NONOMATCH in effect, and highlighters/*.zsh has no matches.
+	for dirname in highlighters highlighters/*/ ; do \
+		$(INSTALL) -d $(SHARE_DIR)/"$$dirname"; \
+		for fname in "$$dirname"/*.zsh ; do [ -e "$$fname" ] && cp "$$fname" $(SHARE_DIR)"/$$dirname"; done; \
+	done
+	cp -R docs/* $(DOC_DIR)
+
+clean:
+	rm -f docs/all.md
 
 test:
 	@result=0; \
 	for test in highlighters/*; do \
 		if [ -d $$test/test-data ]; then \
 			echo "Running test $${test##*/}"; \
-			zsh -f tests/test-highlighting.zsh "$${test##*/}"; \
+			$(ZSH) -f tests/test-highlighting.zsh "$${test##*/}"; \
 			: $$(( result |= $$? )); \
 		fi \
 	done; \
 	exit $$result
 
-.PHONY: all install test
+perf:
+	@result=0; \
+	for test in highlighters/*; do \
+		if [ -d $$test/test-data ]; then \
+			echo "Running test $${test##*/}"; \
+			$(ZSH) -f tests/test-perfs.zsh "$${test##*/}"; \
+			: $$(( result |= $$? )); \
+		fi \
+	done; \
+	exit $$result
+
+.PHONY: all install clean test perf

README.md

@@ -1,53 +1,26 @@
 zsh-syntax-highlighting
 =======================
 
-**[Fish shell](http://www.fishshell.com) like syntax highlighting for [Zsh](http://www.zsh.org).**
+**[Fish shell][fish]-like like syntax highlighting for [Zsh][zsh].**
 
 *Requirements: zsh 4.3.17+.*
 
+[fish]: http://www.fishshell.com/
+[zsh]: http://www.zsh.org/
+
+This package provides syntax highlighing for the shell zsh.  It enables
+highlighing of commands whilst they are typed at a zsh prompt into an
+interactive terminal.  This helps in reviewing commands before running
+them, particularly in catching syntax errors.
+
+[![Screenshot](images/preview-smaller.png)](images/preview.png)
+
 
 How to install
 --------------
 
-### Using packages
+See [INSTALL.md](INSTALL.md).
 
-* Arch Linux: [community/zsh-syntax-highlighting](https://www.archlinux.org/packages/zsh-syntax-highlighting) / [AUR/zsh-syntax-highlighting-git](https://aur.archlinux.org/packages/zsh-syntax-highlighting-git)
-* Gentoo: [mv overlay](http://gpo.zugaina.org/app-shells/zsh-syntax-highlighting)
-* Mac OS X / Homebrew: [brew install zsh-syntax-highlighting](https://github.com/Homebrew/homebrew/blob/master/Library/Formula/zsh-syntax-highlighting.rb)
-
-### In your ~/.zshrc
-
-* Clone this repository:
-
-        git clone git://github.com/zsh-users/zsh-syntax-highlighting.git
-
-  (or [download a snapshot](https://github.com/zsh-users/zsh-syntax-highlighting/archive/master.tar.gz))
-
-* Source the script **at the end** of `~/.zshrc`:
-
-        source /path/to/zsh-syntax-highlighting/zsh-syntax-highlighting.zsh
-
-* Source `~/.zshrc`  to take changes into account:
-
-        source ~/.zshrc
-
-
-### With oh-my-zsh
-
-* Download the script or clone this repository in [oh-my-zsh](http://github.com/robbyrussell/oh-my-zsh) plugins directory:
-
-        git clone git://github.com/zsh-users/zsh-syntax-highlighting.git ~/.oh-my-zsh/custom/plugins/zsh-syntax-highlighting
-
-* Activate the plugin in `~/.zshrc`:
-
-        plugins=( [plugins...] zsh-syntax-highlighting)
-
-* Source `~/.zshrc`  to take changes into account:
-
-        source ~/.zshrc
-
-Note that `zsh-syntax-highlighting` must be the last plugin sourced,
-so make it the last element of the `$plugins` array.
 
 FAQ
 ---
@@ -61,13 +34,16 @@ syntax highlighting.
 
 ### How are new releases announced?
 
-There is currently no "push" announcements channel.  However, the following alternatives exist:
+There is currently no "push" announcements channel.  However, the following
+alternatives exist:
 
 - GitHub's RSS feed of releases: https://github.com/zsh-users/zsh-syntax-highlighting/releases.atom
 - An anitya entry: https://release-monitoring.org/project/7552/
 
+
 How to tweak
 ------------
 
-Syntax highlighting is done by pluggable highlighter scripts, see the [highlighters directory](highlighters)
-for documentation and configuration settings.
+Syntax highlighting is done by pluggable highlighter scripts.  See the
+[documentation on highlighters](docs/highlighters.md) for details and
+configuration settings.

changelog.md

@@ -1,3 +1,175 @@
+
+
+
+
+# Changes in version 0.4.1
+
+## Fixes:
+
+- Arguments to widgets were not properly dash-escaped.  Only matters for widgets
+  that take arguments (i.e., that are invoked as `zle ${widget} -- ${args}`).
+  (282c7134e8ac, reverts c808d2187a73)
+
+
+# Changes in version 0.4.0
+
+
+## Added highlighting of:
+
+- incomplete sudo commands
+  (a3047a912100, 2f05620b19ae)
+
+        sudo;
+        sudo -u;
+
+- command words following reserved words
+  (#207, #222, b397b12ac139 et seq, 6fbd2aa9579b et seq, 8b4adbd991b0)
+
+        if ls; then ls; else ls; fi
+        repeat 10 do ls; done
+
+    (The `ls` are now highlighted as a command.)
+
+- comments (when `INTERACTIVE_COMMENTS` is set)
+  (#163, #167, 693de99a9030)
+
+        echo Hello # comment
+
+- closing brackets of arithmetic expansion, subshells, and blocks
+  (#226, a59f442d2d34, et seq)
+
+        (( foo ))
+        ( foo )
+        { foo }
+
+- command names enabled by the `PATH_DIRS` option
+  (#228, 96ee5116b182)
+
+        # When ~/bin/foo/bar exists, is executable, ~/bin is in $PATH,
+        # and 'setopt PATH_DIRS' is in effect
+        foo/bar
+
+- parameter expansions with braces inside double quotes
+  (#186, 6e3720f39d84)
+
+        echo "${foo}"
+
+- parameter expansions in command word
+  (#101, 4fcfb15913a2)
+
+        x=/bin/ls
+        $x -l
+
+- the command separators '|&', '&!', '&|'
+
+        view file.pdf &!  ls
+
+
+## Fixed highlighting of:
+
+- precommand modifiers at non-command-word position
+  (#209, 2c9f8c8c95fa)
+
+        ls command foo
+
+- sudo commands with infix redirections
+  (#221, be006aded590, 86e924970911)
+
+        sudo -u >/tmp/foo.out user ls
+
+- subshells; anonymous functions
+  (#166, #194, 0d1bfbcbfa67, 9e178f9f3948)
+
+        (true)
+        () { true }
+
+- parameter assignment statements with no command
+  (#205, 01d7eeb3c713)
+
+        A=1;
+
+    (The semicolon used to be highlighted as a mistake)
+
+- cursor highlighter: Remove the cursor highlighting when accepting a line.
+  (#109, 4f0c293fdef0)
+
+
+## Removed features:
+
+- Removed highlighting of approximate paths (`path_approx`).
+  (#187, 98aee7f8b9a3)
+
+
+## Other changes:
+
+- main highlighter refactored to use states rather than booleans.
+  (2080a441ac49, et seq)
+
+- Fix initialization when sourcing `zsh-syntax-highlighting.zsh` via a symlink
+  (083c47b00707)
+
+- docs: Add screenshot.
+  (57624bb9f64b)
+
+- widgets wrapping: Don't add '--' when invoking widgets.
+  (c808d2187a73) [_reverted in 0.4.1_]
+
+- Refresh highlighting upon `accept-*` widgets (`accept-line` et al).
+  (59fbdda64c21)
+
+- Stop leaking match/mbegin/mend to global scope (thanks to upstream
+  `WARN_CREATE_GLOBAL` improvements).
+  (d3deffbf46a4)
+
+- 'make install': Permit setting `$(SHARE_DIR)` from the environment.
+  (e1078a8b4cf1)
+
+- driver: Tolerate KSH_ARRAYS being set in the calling context.
+  (#162, 8f19af6b319d)
+
+- 'make install': Install documentation fully and properly.
+  (#219, b1619c001390, et seq)
+
+- docs: Improve 'main' highlighter's documentation.
+  (00de155063f5, 7d4252f5f596)
+
+- docs: Moved to a new docs/ tree; assorted minor updates
+  (c575f8f37567, 5b34c23cfad5, et seq)
+
+- docs: Split README.md into INSTALL.md
+  (0b3183f6cb9a)
+
+- driver: Report `$ZSH_HIGHLIGHT_REVISION` when running from git
+  (84734ba95026)
+
+
+## Developer-visible changes:
+
+- Test harness converted to [TAP](http://testanything.org/tap-specification.html) format
+  (d99aa58aaaef, et seq)
+
+- Run each test in a separate subprocess, isolating them from each other
+  (d99aa58aaaef, et seq)
+
+- Fix test failure with nonexisting $HOME
+  (#216, b2ac98b98150)
+
+- Test output is now colorized.
+  (4d3da30f8b72, 6fe07c096109)
+
+- Document `make install`
+  (a18a7427fd2c)
+
+- tests: Allow specifying the zsh binary to use.
+  (557bb7e0c6a0)
+
+- tests: Add 'make perf' target
+  (4513eaea71d7)
+
+- tests: Run each test in a sandbox directory
+  (c01533920245)
+
+
 # Changes in version 0.3.0
 
 

docs/highlighters.md

@@ -0,0 +1,77 @@
+zsh-syntax-highlighting / highlighters
+======================================
+
+Syntax highlighting is done by pluggable highlighters:
+
+* `main` - the base highlighter, and the only one [active by default][1].
+* `brackets` - [matches brackets][2] and parenthesis.
+* `pattern` - matches [user-defined patterns][3].
+* `cursor` - matches [the cursor position][4].
+* `root` - highlights the whole command line [if the current user is root][5].
+* `line` - applied to [the whole command line][6].
+
+[1]: highlighters/main.md
+[2]: highlighters/brackets.md
+[3]: highlighters/pattern.md
+[4]: highlighters/cursor.md
+[5]: highlighters/root.md
+[6]: highlighters/line.md
+
+
+How to activate highlighters
+----------------------------
+
+To activate an highlighter, add it to the `ZSH_HIGHLIGHT_HIGHLIGHTERS` array in
+`~/.zshrc`, for example:
+
+    ZSH_HIGHLIGHT_HIGHLIGHTERS=(main brackets pattern cursor)
+
+By default, `$ZSH_HIGHLIGHT_HIGHLIGHTERS` is unset and only the `main`
+highlighter is active.
+
+
+How to tweak highlighters
+-------------------------
+
+Highlighters look up styles from the `ZSH_HIGHLIGHT_STYLES` associative array.
+Navigate into the [individual highlighters' documentation](highlighters/) to
+see what styles (keys) each highlighter defines; the syntax for values is the
+same as the syntax of "types of highlighting" of the zsh builtin
+`$zle_highlight` array, which is documented in [the `zshzle(1)` manual
+page][zshzle-Character-Highlighting].
+
+[zshzle-Character-Highlighting]: http://zsh.sourceforge.net/Doc/Release/Zsh-Line-Editor.html#Character-Highlighting
+
+Some highlighters support additional configuration parameters; see each
+highlighter's documentation for details and examples.
+
+
+How to implement a new highlighter
+----------------------------------
+
+To create your own `myhighlighter` highlighter:
+
+* Create your script at
+    `highlighters/${myhighlighter}/${myhighlighter}-highlighter.zsh`.
+
+* Implement the `_zsh_highlight_myhighlighter_highlighter_predicate` function.
+  This function must return 0 when the highlighter needs to be called and
+  non-zero otherwise, for example:
+
+        _zsh_highlight_myhighlighter_highlighter_predicate() {
+          # Call this highlighter in SVN working copies
+          [[ -d .svn ]]
+        }
+
+* Implement the `_zsh_highlight_myhighlighter_highlighter` function.
+  This function does the actual syntax highlighting, by modifying
+  `region_highlight`, for example:
+
+        _zsh_highlight_myhighlighter_highlighter() {
+          # Colorize the whole buffer with blue background
+          region_highlight+=(0 $#BUFFER bg=blue)
+        }
+
+* Activate your highlighter in `~/.zshrc`:
+
+        ZSH_HIGHLIGHT_HIGHLIGHTERS+=(myhighlighter)

docs/highlighters/brackets.md

@@ -0,0 +1,29 @@
+zsh-syntax-highlighting / highlighters / brackets
+-------------------------------------------------
+
+This is the `brackets` highlighter, that highlights brackets and parentheses, and
+matches them.
+
+
+### How to tweak it
+
+This highlighter defines the following styles:
+
+* `bracket-error` - unmatched brackets
+* `bracket-level-N` - brackets with nest level N
+* `cursor-matchingbracket` - the matching bracket, if cursor is on a bracket
+
+To override one of those styles, change its entry in `ZSH_HIGHLIGHT_STYLES`,
+for example in `~/.zshrc`:
+
+    # To define styles for nested brackets up to level 4
+    ZSH_HIGHLIGHT_STYLES[bracket-level-1]='fg=blue,bold'
+    ZSH_HIGHLIGHT_STYLES[bracket-level-2]='fg=red,bold'
+    ZSH_HIGHLIGHT_STYLES[bracket-level-3]='fg=yellow,bold'
+    ZSH_HIGHLIGHT_STYLES[bracket-level-4]='fg=magenta,bold'
+
+The syntax for values is the same as the syntax of "types of highlighting" of
+the zsh builtin `$zle_highlight` array, which is documented in [the `zshzle(1)`
+manual page][zshzle-Character-Highlighting].
+
+[zshzle-Character-Highlighting]: http://zsh.sourceforge.net/Doc/Release/Zsh-Line-Editor.html#Character-Highlighting

docs/highlighters/cursor.md

@@ -0,0 +1,22 @@
+zsh-syntax-highlighting / highlighters / cursor
+-----------------------------------------------
+
+This is the `cursor` highlighter, that highlights the cursor.
+
+
+### How to tweak it
+
+This highlighter defines the following styles:
+
+* `cursor` - the style for the current cursor position
+
+To override one of those styles, change its entry in `ZSH_HIGHLIGHT_STYLES`,
+for example in `~/.zshrc`:
+
+    ZSH_HIGHLIGHT_STYLES[cursor]='bg=blue'
+
+The syntax for values is the same as the syntax of "types of highlighting" of
+the zsh builtin `$zle_highlight` array, which is documented in [the `zshzle(1)`
+manual page][zshzle-Character-Highlighting].
+
+[zshzle-Character-Highlighting]: http://zsh.sourceforge.net/Doc/Release/Zsh-Line-Editor.html#Character-Highlighting

docs/highlighters/line.md

@@ -0,0 +1,22 @@
+zsh-syntax-highlighting / highlighters / line
+---------------------------------------------
+
+This is the `line` highlighter, that highlights the whole line.
+
+
+### How to tweak it
+
+This highlighter defines the following styles:
+
+* `line` - the style for the whole line
+
+To override one of those styles, change its entry in `ZSH_HIGHLIGHT_STYLES`,
+for example in `~/.zshrc`:
+
+    ZSH_HIGHLIGHT_STYLES[line]='bold'
+
+The syntax for values is the same as the syntax of "types of highlighting" of
+the zsh builtin `$zle_highlight` array, which is documented in [the `zshzle(1)`
+manual page][zshzle-Character-Highlighting].
+
+[zshzle-Character-Highlighting]: http://zsh.sourceforge.net/Doc/Release/Zsh-Line-Editor.html#Character-Highlighting

docs/highlighters/main.md

@@ -0,0 +1,66 @@
+zsh-syntax-highlighting / highlighters / main
+---------------------------------------------
+
+This is the `main` highlighter, that highlights:
+
+* Commands
+* Options
+* Arguments
+* Paths
+* Strings
+
+This highlighter is active by default.
+
+
+### How to tweak it
+
+This highlighter defines the following styles:
+
+* `unknown-token` - unknown tokens / errors
+* `reserved-word` - shell reserved words (`if`, `for`)
+* `alias` - aliases
+* `suffix-alias` - suffix aliases (requires zsh 5.1.1 or newer)
+* `builtin` - shell builtin commands (`shift`, `pwd`, `zstyle`)
+* `function` - function names
+* `command` - command names
+* `precommand` - precommand modifiers (e.g., `noglob`, `builtin`)
+* `commandseparator` - command separation tokens (`;`, `&&`)
+* `hashed-command` - hashed commands
+* `path` - existing filenames
+* `path_prefix` - prefixes of existing filenames
+* `globbing` - globbing expressions (`*.txt`)
+* `history-expansion` - history expansion expressions (`!foo` and `^foo^bar`)
+* `single-hyphen-option` - single hyphen options (`-o`)
+* `double-hyphen-option` - double hyphen options (`--option`)
+* `back-quoted-argument` - backquoted expressions (`` `foo` ``)
+* `single-quoted-argument` - single quoted arguments (`` 'foo' ``)
+* `double-quoted-argument` - double quoted arguments (`` "foo" ``)
+* `dollar-quoted-argument` - dollar quoted arguments (`` $'foo' ``)
+* `dollar-double-quoted-argument` - parameter expansion inside double quotes (`$foo` inside `""`)
+* `back-double-quoted-argument` -  back double quoted arguments (`\x` inside `""`)
+* `back-dollar-quoted-argument` -  back dollar quoted arguments (`\x` inside `$''`)
+* `assign` - parameter assignments
+* `redirection` - redirection operators (`<`, `>`, etc)
+* `comment` - comments, when `setopt INTERACTIVE_COMMENTS` is in effect (`echo # foo`)
+* `default` - everything else
+
+To override one of those styles, change its entry in `ZSH_HIGHLIGHT_STYLES`,
+for example in `~/.zshrc`:
+
+    # Declare the variable
+    typeset -A ZSH_HIGHLIGHT_STYLES
+
+    # To differentiate aliases from other command types
+    ZSH_HIGHLIGHT_STYLES[alias]='fg=magenta,bold'
+    
+    # To have paths colored instead of underlined
+    ZSH_HIGHLIGHT_STYLES[path]='fg=cyan'
+    
+    # To disable highlighting of globbing expressions
+    ZSH_HIGHLIGHT_STYLES[globbing]='none'
+
+The syntax for values is the same as the syntax of "types of highlighting" of
+the zsh builtin `$zle_highlight` array, which is documented in [the `zshzle(1)`
+manual page][zshzle-Character-Highlighting].
+
+[zshzle-Character-Highlighting]: http://zsh.sourceforge.net/Doc/Release/Zsh-Line-Editor.html#Character-Highlighting

docs/highlighters/pattern.md

@@ -0,0 +1,19 @@
+zsh-syntax-highlighting / highlighters / pattern
+------------------------------------------------
+
+This is the `pattern` highlighter, that highlights user-defined patterns.
+
+
+### How to tweak it
+
+To use this highlighter, associate patterns with styles in the
+`ZSH_HIGHLIGHT_PATTERNS` associative array, for example in `~/.zshrc`:
+
+    # To have commands starting with `rm -rf` in red:
+    ZSH_HIGHLIGHT_PATTERNS+=('rm -rf *' 'fg=white,bold,bg=red')
+
+The syntax for values is the same as the syntax of "types of highlighting" of
+the zsh builtin `$zle_highlight` array, which is documented in [the `zshzle(1)`
+manual page][zshzle-Character-Highlighting].
+
+[zshzle-Character-Highlighting]: http://zsh.sourceforge.net/Doc/Release/Zsh-Line-Editor.html#Character-Highlighting

docs/highlighters/root.md

@@ -0,0 +1,23 @@
+zsh-syntax-highlighting / highlighters / root
+---------------------------------------------
+
+This is the `root` highlighter, that highlights the whole line if the current
+user is root.
+
+
+### How to tweak it
+
+This highlighter defines the following styles:
+
+* `root` - the style for the whole line if the current user is root.
+
+To override one of those styles, change its entry in `ZSH_HIGHLIGHT_STYLES`,
+for example in `~/.zshrc`:
+
+    ZSH_HIGHLIGHT_STYLES[root]='bg=red'
+
+The syntax for values is the same as the syntax of "types of highlighting" of
+the zsh builtin `$zle_highlight` array, which is documented in [the `zshzle(1)`
+manual page][zshzle-Character-Highlighting].
+
+[zshzle-Character-Highlighting]: http://zsh.sourceforge.net/Doc/Release/Zsh-Line-Editor.html#Character-Highlighting

highlighters/README.md

@@ -1,50 +1,8 @@
 zsh-syntax-highlighting / highlighters
 ======================================
 
-Syntax highlighting is done by pluggable highlighters:
+Navigate into the individual highlighters' documentation to see
+what styles (`$ZSH_HIGHLIGHT_STYLES` keys) each highlighter defines.
 
-* [***main***](main) - the base highlighter, and the only one active by default.
-* [***brackets***](brackets) - matches brackets and parenthesis.
-* [***pattern***](pattern) - matches user-defined patterns.
-* [***cursor***](cursor) - matches the cursor position.
-* [***root***](root) - triggered if the current user is root.
-* [***line***](line) - applied to the whole command line
-
-
-How to activate highlighters
-----------------------------
-
-To activate an highlighter, add it to the `ZSH_HIGHLIGHT_HIGHLIGHTERS` array in `~/.zshrc`, for example:
-
-    ZSH_HIGHLIGHT_HIGHLIGHTERS=(main brackets pattern cursor)
-
-
-How to tweak highlighters
--------------------------
-
-Highlighters look up styles from the `ZSH_HIGHLIGHT_STYLES` array. Navigate into each highlighter directory to see what styles it defines and how to configure it.
-
-
-How to implement a new highlighter
-----------------------------------
-
-To create your own ***myhighlighter*** highlighter:
-
-* Create your script at **highlighters/*myhighlighter*/*myhighlighter*-highlighter.zsh**.
-* Implement the `_zsh_highlight_myhighlighter_highlighter_predicate` function. This function must return 0 when the highlighter needs to be called, for example:
-
-        _zsh_highlight_myhighlighter_highlighter_predicate() {
-          # Call this highlighter in SVN repositories
-          [[ -d .svn ]]
-        }
-
-* Implement the `_zsh_highlight_myhighlighter_highlighter` function. This function does the actual syntax highlighting, by modifying `region_highlight`, for example:
-
-        _zsh_highlight_myhighlighter_highlighter() {
-          # Colorize the whole buffer with blue background
-          region_highlight+=(0 $#BUFFER bg=blue)
-        }
-
-* Activate your highlighter in `~/.zshrc`:
-
-        ZSH_HIGHLIGHT_HIGHLIGHTERS+=(myhighlighter)
+Refer to the [documentation on highlighters](../docs/highlighters.md) for further
+information.

highlighters/brackets/README.md

@@ -1,30 +0,0 @@
-zsh-syntax-highlighting / highlighters / brackets
-=================================================
-
-This is the ***brackets*** highlighter, that highlights brackets, parenthesis and matches them.
-
-
-How to activate it
-------------------
-To activate it, add it to `ZSH_HIGHLIGHT_HIGHLIGHTERS`:
-
-    ZSH_HIGHLIGHT_HIGHLIGHTERS=( [...] brackets)
-
-
-How to tweak it
----------------
-This highlighter defines the following styles:
-
-* `bracket-error` - unmatched brackets
-* `bracket-level-N` - brackets with nest level N
-* `cursor-matchingbracket` - the matching bracket, if cursor is on a bracket
-
-To override one of those styles, change its entry in `ZSH_HIGHLIGHT_STYLES`, for example in `~/.zshrc`:
-
-    # To define styles for nested brackets up to level 4
-    ZSH_HIGHLIGHT_STYLES[bracket-level-1]='fg=blue,bold'
-    ZSH_HIGHLIGHT_STYLES[bracket-level-2]='fg=red,bold'
-    ZSH_HIGHLIGHT_STYLES[bracket-level-3]='fg=yellow,bold'
-    ZSH_HIGHLIGHT_STYLES[bracket-level-4]='fg=magenta,bold'
-
-The syntax for declaring styles is [documented here](http://zsh.sourceforge.net/Doc/Release/Zsh-Line-Editor.html#SEC135).

highlighters/brackets/README.md

@@ -0,0 +1 @@
+../../docs/highlighters/brackets.md

highlighters/cursor/README.md

@@ -1,24 +0,0 @@
-zsh-syntax-highlighting / highlighters / cursor
-=================================================
-
-This is the ***cursor*** highlighter, that highlights the cursor.
-
-
-How to activate it
-------------------
-To activate it, add it to `ZSH_HIGHLIGHT_HIGHLIGHTERS`:
-
-    ZSH_HIGHLIGHT_HIGHLIGHTERS=( [...] cursor)
-
-
-How to tweak it
----------------
-This highlighter defines the following styles:
-
-* `cursor` - the style for the current cursor position
-
-To override one of those styles, change its entry in `ZSH_HIGHLIGHT_STYLES`, for example in `~/.zshrc`:
-
-    ZSH_HIGHLIGHT_STYLES[cursor]='bg=blue'
-
-The syntax for declaring styles is [documented here](http://zsh.sourceforge.net/Doc/Release/Zsh-Line-Editor.html#SEC135).

highlighters/cursor/README.md

@@ -0,0 +1 @@
+../../docs/highlighters/cursor.md

highlighters/cursor/cursor-highlighter.zsh

@@ -34,11 +34,15 @@
 # Whether the cursor highlighter should be called or not.
 _zsh_highlight_cursor_highlighter_predicate()
 {
+  # accept-* may trigger removal of cursor highlighting
+  [[ $WIDGET == accept-* ]] ||
     _zsh_highlight_cursor_moved
 }
 
 # Cursor highlighting function.
 _zsh_highlight_cursor_highlighter()
 {
+  [[ $WIDGET == accept-* ]] && return
+  
   region_highlight+=("$CURSOR $(( $CURSOR + 1 )) $ZSH_HIGHLIGHT_STYLES[cursor]")
 }

highlighters/line/README.md

@@ -1,24 +0,0 @@
-zsh-syntax-highlighting / highlighters / line
-=================================================
-
-This is the ***line*** highlighter, that highlights the whole line.
-
-
-How to activate it
-------------------
-To activate it, add it to `ZSH_HIGHLIGHT_HIGHLIGHTERS`:
-
-    ZSH_HIGHLIGHT_HIGHLIGHTERS=( [...] line)
-
-
-How to tweak it
----------------
-This highlighter defines the following styles:
-
-* `line` - the style for the whole line
-
-To override one of those styles, change its entry in `ZSH_HIGHLIGHT_STYLES`, for example in `~/.zshrc`:
-
-    ZSH_HIGHLIGHT_STYLES[line]='bold'
-
-The syntax for declaring styles is [documented here](http://zsh.sourceforge.net/Doc/Release/Zsh-Line-Editor.html#SEC135).

highlighters/line/README.md

@@ -0,0 +1 @@
+../../docs/highlighters/line.md

highlighters/main/README.md

@@ -1,68 +0,0 @@
-zsh-syntax-highlighting / highlighters / main
-=============================================
-
-This is the ***main*** highlighter, that highlights:
-
-* Commands
-* Options
-* Arguments
-* Paths
-* Strings
-
-How to activate it
-------------------
-To activate it, add it to `ZSH_HIGHLIGHT_HIGHLIGHTERS`:
-
-    ZSH_HIGHLIGHT_HIGHLIGHTERS=( [...] main)
-
-This highlighter is active by default.
-
-
-How to tweak it
----------------
-This highlighter defines the following styles:
-
-* `unknown-token` - unknown tokens / errors
-* `reserved-word` - shell reserved words
-* `alias` - aliases
-* `suffix-alias` - suffix aliases (requires zsh 5.1.1 or newer)
-* `builtin` - shell builtin commands
-* `function` - functions
-* `command` - commands
-* `precommand` - precommands (i.e. exec, builtin, ...)
-* `commandseparator` - command separation tokens
-* `hashed-command` - hashed commands
-* `path` - paths
-* `path_prefix` - path prefixes
-* `path_approx` - approximated paths
-* `globbing` - globbing expressions (`*.txt`)
-* `history-expansion` - history expansion expressions (`!foo` and `^foo^bar`)
-* `single-hyphen-option` - single hyphen options (-o)
-* `double-hyphen-option` - double hyphen options (--option)
-* `back-quoted-argument` - backquoted expressions (`` `foo` ``)
-* `single-quoted-argument` - single quoted arguments (`` 'foo' ``)
-* `double-quoted-argument` - double quoted arguments (`` "foo" ``)
-* `dollar-quoted-argument` - dollar quoted arguments (`` $'foo' ``)
-* `dollar-double-quoted-argument` -  dollar double quoted arguments ($foo inside "")
-* `back-double-quoted-argument` -  back double quoted arguments (\x inside "")
-* `back-dollar-quoted-argument` -  back dollar quoted arguments (\x inside $'')
-* `assign` - variable assignments
-* `redirection` - redirection operators (`<`, `>`, etc)
-* `default` - parts of the buffer that do not match anything
-
-To override one of those styles, change its entry in `ZSH_HIGHLIGHT_STYLES`, for example in `~/.zshrc`:
-
-    # Declare the variable
-    typeset -A ZSH_HIGHLIGHT_STYLES
-
-    # To differentiate aliases from other command types
-    ZSH_HIGHLIGHT_STYLES[alias]='fg=magenta,bold'
-    
-    # To have paths colored instead of underlined
-    ZSH_HIGHLIGHT_STYLES[path]='fg=cyan'
-    
-    # To disable highlighting of globbing expressions
-    ZSH_HIGHLIGHT_STYLES[globbing]='none'
-
-The syntax for declaring styles is documented in [the `zshzle(1)` manual
-page](http://zsh.sourceforge.net/Doc/Release/Zsh-Line-Editor.html#SEC135).

highlighters/main/README.md

@@ -0,0 +1 @@
+../../docs/highlighters/main.md

highlighters/main/main-highlighter.zsh

@@ -42,7 +42,6 @@
 : ${ZSH_HIGHLIGHT_STYLES[hashed-command]:=fg=green}
 : ${ZSH_HIGHLIGHT_STYLES[path]:=underline}
 : ${ZSH_HIGHLIGHT_STYLES[path_prefix]:=underline}
-: ${ZSH_HIGHLIGHT_STYLES[path_approx]:=fg=yellow,underline}
 : ${ZSH_HIGHLIGHT_STYLES[globbing]:=fg=blue}
 : ${ZSH_HIGHLIGHT_STYLES[history-expansion]:=fg=blue}
 : ${ZSH_HIGHLIGHT_STYLES[single-hyphen-option]:=none}
@@ -56,10 +55,13 @@
 : ${ZSH_HIGHLIGHT_STYLES[back-dollar-quoted-argument]:=fg=cyan}
 : ${ZSH_HIGHLIGHT_STYLES[assign]:=none}
 : ${ZSH_HIGHLIGHT_STYLES[redirection]:=none}
+: ${ZSH_HIGHLIGHT_STYLES[comment]:=fg=black,bold}
 
 # Whether the highlighter should be called or not.
 _zsh_highlight_main_highlighter_predicate()
 {
+  # accept-* may trigger removal of path_prefix highlighting
+  [[ $WIDGET == accept-* ]] ||
     _zsh_highlight_buffer_modified
 }
 
@@ -78,40 +80,138 @@ _zsh_highlight_main_add_region_highlight() {
   region_highlight+=("$start $end $style")
 }
 
+# Wrapper around 'type -w'.
+#
+# Takes a single argument and outputs the output of 'type -w $1'.
+#
+# NOTE: This runs 'setopt', but that should be safe since it'll only ever be
+# called inside a $(...) subshell, so the effects will be local.
+_zsh_highlight_main__type() {
+  if (( $#options_to_set )); then
+    setopt $options_to_set;
+  fi
+  LC_ALL=C builtin type -w -- $1 2>/dev/null
+}
+
 # Main syntax highlighting function.
 _zsh_highlight_main_highlighter()
 {
+  ## Before we even 'emulate -L', we must test a few options that would reset.
+  if [[ -o interactive_comments ]]; then
+    local interactive_comments= # set to empty
+  fi
+  if [[ -o path_dirs ]]; then
+    integer path_dirs_was_set=1
+  else
+    integer path_dirs_was_set=0
+  fi
   emulate -L zsh
   setopt localoptions extendedglob bareglobqual
-  local start_pos=0 end_pos highlight_glob=true new_expression=true arg style sudo=false sudo_arg=false
-  local redirection=false # true when we've seen a redirection operator before seeing the command word
+
+  ## Variable declarations and initializations
+  local start_pos=0 end_pos highlight_glob=true arg style
+  local in_array_assignment=false # true between 'a=(' and the matching ')'
   typeset -a ZSH_HIGHLIGHT_TOKENS_COMMANDSEPARATOR
   typeset -a ZSH_HIGHLIGHT_TOKENS_PRECOMMANDS
-  typeset -a ZSH_HIGHLIGHT_TOKENS_FOLLOWED_BY_COMMANDS
+  typeset -a ZSH_HIGHLIGHT_TOKENS_CONTROL_FLOW
+  local -a options_to_set # used in callees
   local buf="$PREBUFFER$BUFFER"
   region_highlight=()
 
+  if (( path_dirs_was_set )); then
+    options_to_set+=( PATH_DIRS )
+  fi
+  unset path_dirs_was_set
+
   ZSH_HIGHLIGHT_TOKENS_COMMANDSEPARATOR=(
     '|' '||' ';' '&' '&&'
+    '|&'
+    '&!' '&|'
+    # ### 'case' syntax, but followed by a pattern, not by a command
+    # ';;' ';&' ';|'
   )
   ZSH_HIGHLIGHT_TOKENS_PRECOMMANDS=(
     'builtin' 'command' 'exec' 'nocorrect' 'noglob'
   )
-  # Tokens that are always immediately followed by a command.
-  ZSH_HIGHLIGHT_TOKENS_FOLLOWED_BY_COMMANDS=(
-    $ZSH_HIGHLIGHT_TOKENS_COMMANDSEPARATOR $ZSH_HIGHLIGHT_TOKENS_PRECOMMANDS
+
+  # Tokens that, at (naively-determined) "command position", are followed by
+  # a de jure command position.  All of these are reserved words.
+  ZSH_HIGHLIGHT_TOKENS_CONTROL_FLOW=(
+    $'\x7b' # block
+    $'\x28' # subshell
+    '()' # anonymous function
+    'while'
+    'until'
+    'if'
+    'then'
+    'elif'
+    'else'
+    'do'
+    'time'
+    'coproc'
+    '!' # reserved word; unrelated to $histchars[1]
   )
 
-  for arg in ${(z)buf}; do
-    # substr_color is set to 1 to disable adding an entry to region_highlight
+  # State machine
+  #
+  # The states are:
+  # - :start:      Command word
+  # - :sudo_opt:   A leading-dash option to sudo (such as "-u" or "-i")
+  # - :sudo_arg:   The argument to a sudo leading-dash option that takes one,
+  #                when given as a separate word; i.e., "foo" in "-u foo" (two
+  #                words) but not in "-ufoo" (one word).
+  # - :regular:    "Not a command word", and command delimiters are permitted.
+  #                Mainly used to detect premature termination of commands.
+  #
+  # When the kind of a word is not yet known, $this_word / $next_word may contain
+  # multiple states.  For example, after "sudo -i", the next word may be either
+  # another --flag or a command name, hence the state would include both :start:
+  # and :sudo_opt:.
+  #
+  # The tokens are always added with both leading and trailing colons to serve as
+  # word delimiters (an improvised array); [[ $x == *:foo:* ]] and x=${x//:foo:/} 
+  # will DTRT regardless of how many elements or repetitions $x has..
+  #
+  # Handling of redirections: upon seeing a redirection token, we must stall
+  # the current state --- that is, the value of $this_word --- for two iterations
+  # (one for the redirection operator, one for the word following it representing
+  # the redirection target).  Therefore, we set $in_redirection to 2 upon seeing a
+  # redirection operator, decrement it each iteration, and stall the current state
+  # when it is non-zero.  Thus, upon reaching the next word (the one that follows
+  # the redirection operator and target), $this_word will still contain values
+  # appropriate for the word immediately following the word that preceded the
+  # redirection operator.
+  #
+  # The "the previous word was a redirection operator" state is not communicated
+  # to the next iteration via $next_word/$this_word as usual, but via
+  # $in_redirection.  The value of $next_word from the iteration that processed
+  # the operator is discarded.
+  #
+  local this_word=':start:' next_word
+  integer in_redirection
+  for arg in ${interactive_comments-${(z)buf}} \
+             ${interactive_comments+${(zZ+c+)buf}}; do
+    if (( in_redirection )); then
+      (( --in_redirection ))
+    fi
+    if (( in_redirection == 0 )); then
+      # Initialize $next_word to its default value.
+      next_word=':regular:'
+    else
+      # Stall $next_word.
+    fi
+    # $already_added is set to 1 to disable adding an entry to region_highlight
     # for this iteration.  Currently, that is done for "" and $'' strings,
     # which add the entry early so escape sequences within the string override
     # the string's color.
-    integer substr_color=0
+    integer already_added=0
     local style_override=""
-    if $new_expression && [[ $arg = 'noglob' ]]; then
+    if [[ $this_word == *':start:'* ]]; then
+      in_array_assignment=false
+      if [[ $arg == 'noglob' ]]; then
         highlight_glob=false
       fi
+    fi
 
     # advance $start_pos, skipping over whitespace in $buf.
     if [[ $arg == ';' ]] ; then
@@ -132,33 +232,62 @@ _zsh_highlight_main_highlighter()
       ((end_pos=$start_pos+${#arg}))
     fi
 
+    if [[ -n ${interactive_comments+'set'} && $arg[1] == $histchars[3] ]]; then
+      if [[ $this_word == *(':regular:'|':start:')* ]]; then
+        style=$ZSH_HIGHLIGHT_STYLES[comment]
+      else
+        style=$ZSH_HIGHLIGHT_STYLES[unknown-token] # prematurely terminated
+      fi
+      _zsh_highlight_main_add_region_highlight $start_pos $end_pos $style
+      already_added=1
+      continue
+    fi
+
     # Parse the sudo command line
-    if $sudo; then
+    if (( ! in_redirection )); then
+      if [[ $this_word == *':sudo_opt:'* ]]; then
         case "$arg" in
           # Flag that requires an argument
-        '-'[Cgprtu]) sudo_arg=true;;
+          '-'[Cgprtu]) this_word=${this_word//:start:/};
+                       next_word=':sudo_arg:';;
           # This prevents misbehavior with sudo -u -otherargument
-        '-'*)        sudo_arg=false;;
-        *)           if $sudo_arg; then
-                       sudo_arg=false
-                     else
-                       sudo=false
-                       new_expression=true; highlight_glob=true
-                     fi
-                     ;;
+          '-'*)        this_word=${this_word//:start:/};
+                       next_word+=':start:';
+                       next_word+=':sudo_opt:';;
+          *)           ;;
         esac
+      elif [[ $this_word == *':sudo_arg:'* ]]; then
+        next_word+=':sudo_opt:'
+        next_word+=':start:'
       fi
-    if $new_expression && ! $redirection; then # $arg is the command word
-      new_expression=false
+    fi
+
+    if [[ $this_word == *':start:'* ]] && (( in_redirection == 0 )); then # $arg is the command word
      if [[ -n ${(M)ZSH_HIGHLIGHT_TOKENS_PRECOMMANDS:#"$arg"} ]]; then
       style=$ZSH_HIGHLIGHT_STYLES[precommand]
      elif [[ "$arg" = "sudo" ]]; then
       style=$ZSH_HIGHLIGHT_STYLES[precommand]
-      sudo=true
+      next_word=${next_word//:regular:/}
+      next_word+=':sudo_opt:'
+      next_word+=':start:'
      else
       _zsh_highlight_main_highlighter_expand_path $arg
       local expanded_arg="$REPLY"
-      local res="$(LC_ALL=C builtin type -w -- ${expanded_arg} 2>/dev/null)"
+      local res="$(_zsh_highlight_main__type ${expanded_arg})"
+      () {
+        # Special-case: command word is '$foo', like that, without braces or anything.
+        #
+        # That's not entirely correct --- if the parameter's value happens to be a reserved
+        # word, the parameter expansion will be highlighted as a reserved word --- but that
+        # incorrectness is outweighed by the usability improvement of permitting the use of
+        # parameters that refer to commands, functions, and builtins.
+        local -a match mbegin mend
+        local MATCH; integer MBEGIN MEND
+        if [[ $res == *': none' ]] && (( ${+parameters} )) &&
+           [[ ${arg[1]} == \$ ]] && [[ ${arg:1} =~ ^([A-Za-z_][A-Za-z0-9_]*|[0-9]+)$ ]]; then
+          res="$(_zsh_highlight_main__type ${(P)MATCH})"
+        fi
+      }
       case $res in
         *': reserved')  style=$ZSH_HIGHLIGHT_STYLES[reserved-word];;
         *': suffix alias')
@@ -166,7 +295,7 @@ _zsh_highlight_main_highlighter()
                         ;;
         *': alias')     style=$ZSH_HIGHLIGHT_STYLES[alias]
                         local aliased_command="${"$(alias -- $arg)"#*=}"
-                        [[ -n ${(M)ZSH_HIGHLIGHT_TOKENS_FOLLOWED_BY_COMMANDS:#"$aliased_command"} && -z ${(M)ZSH_HIGHLIGHT_TOKENS_FOLLOWED_BY_COMMANDS:#"$arg"} ]] && ZSH_HIGHLIGHT_TOKENS_FOLLOWED_BY_COMMANDS+=($arg)
+                        [[ -n ${(M)ZSH_HIGHLIGHT_TOKENS_PRECOMMANDS:#"$aliased_command"} && -z ${(M)ZSH_HIGHLIGHT_TOKENS_PRECOMMANDS:#"$arg"} ]] && ZSH_HIGHLIGHT_TOKENS_PRECOMMANDS+=($arg)
                         ;;
         *': builtin')   style=$ZSH_HIGHLIGHT_STYLES[builtin];;
         *': function')  style=$ZSH_HIGHLIGHT_STYLES[function];;
@@ -174,16 +303,23 @@ _zsh_highlight_main_highlighter()
         *': hashed')    style=$ZSH_HIGHLIGHT_STYLES[hashed-command];;
         *)              if _zsh_highlight_main_highlighter_check_assign; then
                           style=$ZSH_HIGHLIGHT_STYLES[assign]
-                          if [[ $arg[-1] != '(' ]]; then
+                          if [[ $arg[-1] == '(' ]]; then
+                            in_array_assignment=true
+                          else
                             # assignment to a scalar parameter.
                             # (For array assignments, the command doesn't start until the ")" token.)
-                            new_expression=true; highlight_glob=true
+                            next_word+=':start:'
                           fi
                         elif [[ $arg[0,1] == $histchars[0,1] || $arg[0,1] == $histchars[2,2] ]]; then
                           style=$ZSH_HIGHLIGHT_STYLES[history-expansion]
+                        elif [[ -n ${(M)ZSH_HIGHLIGHT_TOKENS_COMMANDSEPARATOR:#"$arg"} ]] &&
+                             [[ $this_word == *':regular:'* ]]; then
+                          # This highlights empty commands (semicolon follows nothing) as an error.
+                          # Zsh accepts them, though.
+                          style=$ZSH_HIGHLIGHT_STYLES[commandseparator]
                         elif [[ $arg[1] == '<' || $arg[1] == '>' ]]; then
                           style=$ZSH_HIGHLIGHT_STYLES[redirection]
-                          redirection=true
+                          (( in_redirection=2 ))
                         elif [[ $arg[1,2] == '((' ]]; then
                           # Arithmetic evaluation.
                           #
@@ -196,7 +332,15 @@ _zsh_highlight_main_highlighter()
                           # is how [[ ... ]] is highlighted, too.
                           style=$ZSH_HIGHLIGHT_STYLES[reserved-word]
                           _zsh_highlight_main_add_region_highlight $start_pos $((start_pos + 2)) $style
-                          substr_color=1
+                          already_added=1
+                          if [[ $arg[-2,-1] == '))' ]]; then
+                            _zsh_highlight_main_add_region_highlight $((end_pos - 2)) $end_pos $style
+                            already_added=1
+                          fi
+                        elif [[ $arg == '()' || $arg == $'\x28' ]]; then
+                          # anonymous function
+                          # subshell
+                          style=$ZSH_HIGHLIGHT_STYLES[reserved-word]
                         else
                           if _zsh_highlight_main_highlighter_check_path; then
                             style=$ZSH_HIGHLIGHT_STYLES[path]
@@ -207,24 +351,28 @@ _zsh_highlight_main_highlighter()
                         ;;
       esac
      fi
-    else # $arg is the file target of a prefix redirection, or a non-command word
-      if $redirection; then
-        redirection=false
-        new_expression=true; highlight_glob=true
-      fi
+    else # $arg is a non-command word
       case $arg in
+        $'\x29') # subshell or end of array assignment
+                 if $in_array_assignment; then
+                   style=$ZSH_HIGHLIGHT_STYLES[assign]
+                   in_array_assignment=false
+                 else
+                   style=$ZSH_HIGHLIGHT_STYLES[reserved-word]
+                 fi;;
+        $'\x7d') style=$ZSH_HIGHLIGHT_STYLES[reserved-word];; # block
         '--'*)   style=$ZSH_HIGHLIGHT_STYLES[double-hyphen-option];;
         '-'*)    style=$ZSH_HIGHLIGHT_STYLES[single-hyphen-option];;
         "'"*)    style=$ZSH_HIGHLIGHT_STYLES[single-quoted-argument];;
         '"'*)    style=$ZSH_HIGHLIGHT_STYLES[double-quoted-argument]
                  _zsh_highlight_main_add_region_highlight $start_pos $end_pos $style
                  _zsh_highlight_main_highlighter_highlight_string
-                 substr_color=1
+                 already_added=1
                  ;;
         \$\'*)   style=$ZSH_HIGHLIGHT_STYLES[dollar-quoted-argument]
                  _zsh_highlight_main_add_region_highlight $start_pos $end_pos $style
                  _zsh_highlight_main_highlighter_highlight_dollar_string
-                 substr_color=1
+                 already_added=1
                  ;;
         '`'*)    style=$ZSH_HIGHLIGHT_STYLES[back-quoted-argument];;
         [*?]*|*[^\\][*?]*)
@@ -233,9 +381,14 @@ _zsh_highlight_main_highlighter()
                  elif [[ $arg[0,1] = $histchars[0,1] ]]; then
                    style=$ZSH_HIGHLIGHT_STYLES[history-expansion]
                  elif [[ -n ${(M)ZSH_HIGHLIGHT_TOKENS_COMMANDSEPARATOR:#"$arg"} ]]; then
+                   if [[ $this_word == *':regular:'* ]]; then
                      style=$ZSH_HIGHLIGHT_STYLES[commandseparator]
+                   else
+                     style=$ZSH_HIGHLIGHT_STYLES[unknown-token]
+                   fi
                  elif [[ $arg[1] == '<' || $arg[1] == '>' ]]; then
                    style=$ZSH_HIGHLIGHT_STYLES[redirection]
+                   (( in_redirection=2 ))
                  else
                    if _zsh_highlight_main_highlighter_check_path; then
                      style=$ZSH_HIGHLIGHT_STYLES[path]
@@ -248,10 +401,25 @@ _zsh_highlight_main_highlighter()
     fi
     # if a style_override was set (eg in _zsh_highlight_main_highlighter_check_path), use it
     [[ -n $style_override ]] && style=$ZSH_HIGHLIGHT_STYLES[$style_override]
-    [[ $substr_color = 0 ]] && _zsh_highlight_main_add_region_highlight $start_pos $end_pos $style
-    [[ -n ${(M)ZSH_HIGHLIGHT_TOKENS_FOLLOWED_BY_COMMANDS:#"$arg"} ]] && new_expression=true
-    [[ -n ${(M)ZSH_HIGHLIGHT_TOKENS_COMMANDSEPARATOR:#"$arg"} ]] && highlight_glob=true
+    (( already_added )) || _zsh_highlight_main_add_region_highlight $start_pos $end_pos $style
+    if [[ -n ${(M)ZSH_HIGHLIGHT_TOKENS_COMMANDSEPARATOR:#"$arg"} ]]; then
+      next_word=':start:'
+      highlight_glob=true
+    elif
+       [[ -n ${(M)ZSH_HIGHLIGHT_TOKENS_CONTROL_FLOW:#"$arg"} && $this_word == *':start:'* ]] ||
+       [[ -n ${(M)ZSH_HIGHLIGHT_TOKENS_PRECOMMANDS:#"$arg"} && $this_word == *':start:'* ]]; then
+      next_word=':start:'
+    elif [[ $arg == "repeat" && $this_word == *':start:'* ]]; then
+      # skip the repeat-count word
+      in_redirection=2
+      # The redirection mechanism assumes $this_word describes the word
+      # following the redirection.  Make it so.
+      #
+      # The repeat-count word will be handled like a redirection target.
+      this_word=':start:'
+    fi
     start_pos=$end_pos
+    (( in_redirection == 0 )) && this_word=$next_word
   done
 }
 
@@ -281,13 +449,11 @@ _zsh_highlight_main_highlighter_check_path()
   [[ ! -e ${expanded_path:h} ]] && return 1
 
   # If this word ends the buffer, check if it's the prefix of a valid path.
-  if [[ ${BUFFER[1]} != "-" && ${#BUFFER} == $end_pos ]]; then
+  if [[ ${BUFFER[1]} != "-" && ${#BUFFER} == $end_pos ]] &&
+     [[ $WIDGET != accept-* ]]; then
     local -a tmp
     tmp=( ${expanded_path}*(N) )
     (( $#tmp > 0 )) && style_override=path_prefix && return 0
-    # or maybe an approximate path?
-    tmp=( (#a1)${expanded_path}*(N) )
-    (( $#tmp > 0 )) && style_override=path_approx && return 0
   fi
 
   # It's not a path.
@@ -298,6 +464,8 @@ _zsh_highlight_main_highlighter_check_path()
 _zsh_highlight_main_highlighter_highlight_string()
 {
   setopt localoptions noksharrays
+  local -a match mbegin mend
+  local MATCH; integer MBEGIN MEND
   local i j k style
   # Starting quote is at 1, so start parsing at offset 2 in the string.
   for (( i = 2 ; i < end_pos - start_pos ; i += 1 )) ; do
@@ -309,6 +477,9 @@ _zsh_highlight_main_highlighter_highlight_string()
             if [[ ${arg:$i} =~ ^([A-Za-z_][A-Za-z0-9_]*|[0-9]+) ]] ; then
               (( k += $#MATCH )) # highlight the parameter name
               (( i += $#MATCH )) # skip past it
+            elif [[ ${arg:$i} =~ ^[{]([A-Za-z_][A-Za-z0-9_]*|[0-9]+)[}] ]] ; then
+              (( k += $#MATCH )) # highlight the parameter name and braces
+              (( i += $#MATCH )) # skip past it
             else
               continue
             fi
@@ -332,6 +503,8 @@ _zsh_highlight_main_highlighter_highlight_string()
 _zsh_highlight_main_highlighter_highlight_dollar_string()
 {
   setopt localoptions noksharrays
+  local -a match mbegin mend
+  local MATCH; integer MBEGIN MEND
   local i j k style
   local AA
   integer c

highlighters/main/test-data/anonymous-function.zsh

@@ -0,0 +1,40 @@
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2015 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER='() echo hello; () { echo world }'
+
+expected_region_highlight=(
+  "1 2 $ZSH_HIGHLIGHT_STYLES[reserved-word]" # () 
+  "4 7 $ZSH_HIGHLIGHT_STYLES[command]" # echo
+  "9 13 $ZSH_HIGHLIGHT_STYLES[default]" # hello
+  "14 14 $ZSH_HIGHLIGHT_STYLES[commandseparator]" # ;
+  "16 17 $ZSH_HIGHLIGHT_STYLES[reserved-word]" # ()
+  "19 19 $ZSH_HIGHLIGHT_STYLES[reserved-word]" # {
+  "21 24 $ZSH_HIGHLIGHT_STYLES[command]" # echo
+)

highlighters/main/test-data/arithmetic-evaluation.zsh

@@ -36,4 +36,5 @@ BUFFER='(( x == 42 ))'
 
 expected_region_highlight=(
   "1 2 $ZSH_HIGHLIGHT_STYLES[reserved-word]" # ((
+  "12 13 $ZSH_HIGHLIGHT_STYLES[reserved-word]" # ))
 )

highlighters/main/test-data/assign-array.zsh

@@ -0,0 +1,38 @@
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2015 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+ZSH_HIGHLIGHT_STYLES[assign]=$unused_highlight
+BUFFER='(A=(hello world))'
+
+expected_region_highlight=(
+  "1 1 $ZSH_HIGHLIGHT_STYLES[reserved-word]" # (
+  "2 4 $ZSH_HIGHLIGHT_STYLES[assign]" # A=(
+  "16 16 $ZSH_HIGHLIGHT_STYLES[assign]" # )
+  "17 17 $ZSH_HIGHLIGHT_STYLES[reserved-word]" # )
+)

highlighters/main/test-data/assign-semicolon.zsh

@@ -0,0 +1,36 @@
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2015 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+ZSH_HIGHLIGHT_STYLES[commandseparator]=$unused_highlight
+BUFFER='A=1; echo hello world'
+
+expected_region_highlight=(
+  "4 4 $ZSH_HIGHLIGHT_STYLES[commandseparator]" # ;
+  "6 9 $ZSH_HIGHLIGHT_STYLES[command]" # echo
+)

highlighters/main/test-data/assign-subshell.zsh

@@ -0,0 +1,37 @@
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2015 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+ZSH_HIGHLIGHT_STYLES[assign]=$unused_highlight
+BUFFER='(A=1)'
+
+expected_region_highlight=(
+  "1 1 $ZSH_HIGHLIGHT_STYLES[reserved-word]" # (
+  "2 4 $ZSH_HIGHLIGHT_STYLES[assign]" # A=1
+  "5 5 $ZSH_HIGHLIGHT_STYLES[reserved-word] 'issue #231'" # )
+)

highlighters/main/test-data/assign.zsh

@@ -32,5 +32,7 @@ BUFFER='A=1 b=("foo" bar)'
 
 expected_region_highlight=(
   "1 3 $ZSH_HIGHLIGHT_STYLES[assign]" # A=1
+  "5 7 $ZSH_HIGHLIGHT_STYLES[assign]" # b=(
   "8 12 $ZSH_HIGHLIGHT_STYLES[double-quoted-argument]" # "foo"
+  "17 17 $ZSH_HIGHLIGHT_STYLES[assign]" # )
 )

highlighters/main/test-data/commandseparator.zsh

@@ -29,9 +29,11 @@
 
 ZSH_HIGHLIGHT_STYLES[commandseparator]=$unused_highlight
 
-BUFFER=':; pwd'
+BUFFER=':; pwd &! ls'
 
 expected_region_highlight=(
   "2 2 $ZSH_HIGHLIGHT_STYLES[commandseparator]" # ;
   "4 6 $ZSH_HIGHLIGHT_STYLES[builtin]" # pwd
+  "8 9 $ZSH_HIGHLIGHT_STYLES[commandseparator]" # &!
+  "11 12 $ZSH_HIGHLIGHT_STYLES[command]" # ls
 )

highlighters/main/test-data/comment-leading.zsh

@@ -0,0 +1,37 @@
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2015 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+setopt interactive_comments
+
+BUFFER='# echo foo'
+
+expected_region_highlight=(
+  "1 1 ${(q-)ZSH_HIGHLIGHT_STYLES[comment]}" # #
+  "2 10 ${(q-)ZSH_HIGHLIGHT_STYLES[comment]}" # " echo foo"
+)

highlighters/main/test-data/comment-off.zsh

@@ -0,0 +1,38 @@
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2015 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+unsetopt interactive_comments
+
+BUFFER='# echo foo'
+
+expected_region_highlight=(
+  "1 1 ${(q-)ZSH_HIGHLIGHT_STYLES[unknown-token]}" # #
+  "3 6 ${(q-)ZSH_HIGHLIGHT_STYLES[default]}" # " echo foo"
+  "8 10 ${(q-)ZSH_HIGHLIGHT_STYLES[default]}" # " echo foo"
+)

highlighters/main/test-data/comments.zsh

@@ -0,0 +1,38 @@
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2015 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+setopt interactive_comments
+
+BUFFER='echo "foo #bar" #baz # quux'
+
+expected_region_highlight=(
+  "1 4 $ZSH_HIGHLIGHT_STYLES[command]" # echo
+  "6 15 $ZSH_HIGHLIGHT_STYLES[double-quoted-argument]" # "foo #bar"
+  "17 27 ${(q-)ZSH_HIGHLIGHT_STYLES[comment]}" # #baz # quux
+)

highlighters/main/test-data/commmand-parameter.zsh

@@ -0,0 +1,36 @@
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2015 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+x=/usr/bin/env
+BUFFER='$x "argument"'
+
+expected_region_highlight=(
+  "1 2 $ZSH_HIGHLIGHT_STYLES[command]" # $x
+  "4 13 $ZSH_HIGHLIGHT_STYLES[double-quoted-argument]" # "argument"
+)

highlighters/main/test-data/control-flow.zsh

@@ -0,0 +1,51 @@
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2015 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+ZSH_HIGHLIGHT_STYLES[alias]=$unused_highlight
+BUFFER='while if echo Hello; then ls /; else ls; fi; do stat "x"; done; repeat 10 ls'
+
+expected_region_highlight+=(
+  "1 5 $ZSH_HIGHLIGHT_STYLES[reserved-word]" # while
+  "7 8 $ZSH_HIGHLIGHT_STYLES[reserved-word]" # if
+  "10 13 $ZSH_HIGHLIGHT_STYLES[builtin]" # echo
+  "15 19 $ZSH_HIGHLIGHT_STYLES[default]" # Hello
+  "22 25 $ZSH_HIGHLIGHT_STYLES[reserved-word]" # then
+  "27 28 $ZSH_HIGHLIGHT_STYLES[command]" # ls
+  "30 30 $ZSH_HIGHLIGHT_STYLES[path]" # /
+  "31 31 $ZSH_HIGHLIGHT_STYLES[commandseparator]" # ;
+  "33 36 $ZSH_HIGHLIGHT_STYLES[reserved-word]" # else
+  "38 39 $ZSH_HIGHLIGHT_STYLES[command]" # ls
+  "42 43 $ZSH_HIGHLIGHT_STYLES[reserved-word]" # fi
+  "46 47 $ZSH_HIGHLIGHT_STYLES[reserved-word]" # do
+  "49 52 $ZSH_HIGHLIGHT_STYLES[command]" # stat
+  "54 56 $ZSH_HIGHLIGHT_STYLES[double-quoted-argument]" # "x"
+  "59 62 $ZSH_HIGHLIGHT_STYLES[reserved-word]" # done
+  "65 70 $ZSH_HIGHLIGHT_STYLES[reserved-word]" # repeat
+  "75 76 $ZSH_HIGHLIGHT_STYLES[command]" # ls
+)

highlighters/main/test-data/double-quoted2.zsh

@@ -1,5 +1,5 @@
 # -------------------------------------------------------------------------------------------------
-# Copyright (c) 2010-2011 zsh-syntax-highlighting contributors
+# Copyright (c) 2015 zsh-syntax-highlighting contributors
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification, are permitted

highlighters/main/test-data/double-quoted3.zsh

@@ -1,5 +1,5 @@
 # -------------------------------------------------------------------------------------------------
-# Copyright (c) 2010-2011 zsh-syntax-highlighting contributors
+# Copyright (c) 2015 zsh-syntax-highlighting contributors
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification, are permitted

highlighters/main/test-data/double-quoted4.zsh

@@ -0,0 +1,36 @@
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2015 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER=': "${foo}bar"'
+
+expected_region_highlight=(
+  "3 3 $ZSH_HIGHLIGHT_STYLES[double-quoted-argument]" # "
+  "4 9 $ZSH_HIGHLIGHT_STYLES[dollar-double-quoted-argument]" # ${foo}
+  "10 13 $ZSH_HIGHLIGHT_STYLES[double-quoted-argument]" # bar"
+)

highlighters/main/test-data/empty-command.zsh

@@ -27,8 +27,7 @@
 # vim: ft=zsh sw=2 ts=2 et
 # -------------------------------------------------------------------------------------------------
 
-# without the trailing space, it's highlighted as path_approx (issue #187, issue #204)
-BUFFER='echo; ; '
+BUFFER='echo; ;'
 
 expected_region_highlight=(
   "5 5 $ZSH_HIGHLIGHT_STYLES[commandseparator]" # ;

highlighters/main/test-data/noglob-alias.zsh

@@ -0,0 +1,35 @@
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2015 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+alias x=command
+BUFFER='x ls'
+
+expected_region_highlight=(
+  "3 4 $ZSH_HIGHLIGHT_STYLES[command]" # ls
+)

highlighters/main/test-data/option-path_dirs.zsh

@@ -0,0 +1,41 @@
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2015 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+## setup
+setopt PATH_DIRS
+mkdir -p foo/bar
+touch foo/bar/testing-issue-228
+chmod  +x foo/bar/testing-issue-228
+path+=( "$PWD"/foo )
+
+BUFFER='bar/testing-issue-228'
+
+expected_region_highlight=(
+  "1 21 $ZSH_HIGHLIGHT_STYLES[command]" # bar/testing-issue-228
+)

highlighters/main/test-data/path-space.zsh

@@ -27,9 +27,11 @@
 # vim: ft=zsh sw=2 ts=2 et
 # -------------------------------------------------------------------------------------------------
 
-BUFFER='ls highlighters/main/test-data/path-space-\ .zsh'
+mkdir A
+touch "A/mu with spaces"
+BUFFER='ls A/mu\ with\ spaces'
 
 expected_region_highlight=(
   "1 2  $ZSH_HIGHLIGHT_STYLES[command]" # ls
-  "4 48 $ZSH_HIGHLIGHT_STYLES[path]"    # highlighters/main/test-data/path-space-\ .zsh
+  "4 19 $ZSH_HIGHLIGHT_STYLES[path]"    # A/mu\ with\ spaces
 )

highlighters/main/test-data/path-tilde-home.zsh

@@ -27,6 +27,7 @@
 # vim: ft=zsh sw=2 ts=2 et
 # -------------------------------------------------------------------------------------------------
 
+HOME="."
 BUFFER='ls ~'
 
 expected_region_highlight=(

highlighters/main/test-data/path-tilde-home2.zsh

@@ -0,0 +1,37 @@
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2015 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+HOME="/nonexistent"
+BUFFER='ls ~'
+
+expected_region_highlight=(
+  "1 2 $ZSH_HIGHLIGHT_STYLES[command]" # ls
+  "4 4 $ZSH_HIGHLIGHT_STYLES[default]"    # ~
+)
+

highlighters/main/test-data/path-tilde-named.zsh

@@ -27,11 +27,13 @@
 # vim: ft=zsh sw=2 ts=2 et
 # -------------------------------------------------------------------------------------------------
 
-hash -d D=highlighters/main/test-data
+mkdir mydir
+touch mydir/path-tilde-named.test
+hash -d D=mydir
 
-BUFFER='ls ~D/path-tilde-named.zsh'
+BUFFER='ls ~D/path-tilde-named.test'
 
 expected_region_highlight=(
   "1 2  $ZSH_HIGHLIGHT_STYLES[command]" # ls
-  "4 26 $ZSH_HIGHLIGHT_STYLES[path]"    # ~D/path-tilde-named.zsh
+  "4 27 $ZSH_HIGHLIGHT_STYLES[path]"    # ~D/path-tilde-named.test
 )

highlighters/main/test-data/path.zsh

@@ -27,9 +27,11 @@
 # vim: ft=zsh sw=2 ts=2 et
 # -------------------------------------------------------------------------------------------------
 
-BUFFER='ls highlighters/main/test-data/path.zsh'
+mkdir A
+touch A/mu
+BUFFER='ls A/mu'
 
 expected_region_highlight=(
   "1 2 $ZSH_HIGHLIGHT_STYLES[command]" # ls
-  "4 39 $ZSH_HIGHLIGHT_STYLES[path]"    # highlighters/main/test-data/path.zsh
+  "4 7 $ZSH_HIGHLIGHT_STYLES[path]"     # A/mu
 )

highlighters/main/test-data/path_prefix.zsh

@@ -27,7 +27,8 @@
 # vim: ft=zsh sw=2 ts=2 et
 # -------------------------------------------------------------------------------------------------
 
-# Assumes that [[ -e /bin/sh ]].
+# Assumes that '/bin/sh' exists and '/bin/s' does not exist.
+# Related to path_prefix2.zsh
 
 ZSH_HIGHLIGHT_STYLES[path_prefix]=$unused_highlight
 BUFFER='ls /bin/s'

highlighters/main/test-data/path_prefix2.zsh

@@ -0,0 +1,39 @@
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2015 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+# Assumes that '/bin/sh' exists and '/bin/s' does not exist.
+# Related to path_prefix.zsh
+
+ZSH_HIGHLIGHT_STYLES[path_prefix]=$unused_highlight
+BUFFER='ls /bin/s'
+WIDGET=accept-line
+
+expected_region_highlight=(
+  "4 9 $ZSH_HIGHLIGHT_STYLES[default]"    # /bin/s
+)

highlighters/main/test-data/precommand.zsh

@@ -0,0 +1,37 @@
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2015 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+ZSH_HIGHLIGHT_STYLES[assign]=$unused_highlight
+BUFFER=': command zzzzzz'
+
+expected_region_highlight=(
+  "1 1 $ZSH_HIGHLIGHT_STYLES[command]" # ls
+  "3 9 $ZSH_HIGHLIGHT_STYLES[default]" # not precommand
+  "11 16 $ZSH_HIGHLIGHT_STYLES[default]" # not unknown-token (since 'zzzzzz' is not a command)
+)

highlighters/main/test-data/prefix-redirection.zsh

@@ -27,12 +27,13 @@
 # vim: ft=zsh sw=2 ts=2 et
 # -------------------------------------------------------------------------------------------------
 
-BUFFER='>/tmp >/tmp sudo echo foo'
+BUFFER='>/tmp >/tmp sudo echo >/tmp foo'
 
 expected_region_highlight=(
   "2  5  $ZSH_HIGHLIGHT_STYLES[path]"       # /tmp
   "8  11 $ZSH_HIGHLIGHT_STYLES[path]"       # /tmp
   "13 16 $ZSH_HIGHLIGHT_STYLES[precommand]" # sudo
   "18 21 $ZSH_HIGHLIGHT_STYLES[builtin]"    # echo
-  "23 25 $ZSH_HIGHLIGHT_STYLES[default]"    # foo
+  "24 27 $ZSH_HIGHLIGHT_STYLES[path]"       # /tmp
+  "29 31 $ZSH_HIGHLIGHT_STYLES[default]"    # foo
 )

highlighters/main/test-data/reserved-word.zsh

@@ -33,4 +33,6 @@ BUFFER='repeat "1" do done'
 expected_region_highlight=(
   "1 6 $ZSH_HIGHLIGHT_STYLES[reserved-word]" # repeat
   "8 10 $ZSH_HIGHLIGHT_STYLES[double-quoted-argument]" # "1"
+  "12 13 $ZSH_HIGHLIGHT_STYLES[reserved-word]" # do
+  "15 18 $ZSH_HIGHLIGHT_STYLES[reserved-word]" # done
 )

highlighters/main/test-data/subshell.zsh

@@ -0,0 +1,41 @@
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2015 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER='tar cf - * | (cd /target; tar xfp -) | { cat }'
+
+expected_region_highlight=(
+  "1 3 $ZSH_HIGHLIGHT_STYLES[command]" # tar
+  "14 14 $ZSH_HIGHLIGHT_STYLES[reserved-word]" # (
+  "15 16 $ZSH_HIGHLIGHT_STYLES[command]" # cd
+  "27 29 $ZSH_HIGHLIGHT_STYLES[command]" # tar
+  "36 36 $ZSH_HIGHLIGHT_STYLES[reserved-word]" # )
+  "40 40 $ZSH_HIGHLIGHT_STYLES[reserved-word]" # {
+  "42 44 $ZSH_HIGHLIGHT_STYLES[command]" # cat
+  "46 46 $ZSH_HIGHLIGHT_STYLES[reserved-word]" # }
+)

highlighters/main/test-data/sudo-command.zsh

@@ -28,12 +28,20 @@
 # -------------------------------------------------------------------------------------------------
 
 ZSH_HIGHLIGHT_STYLES[single-hyphen-option]=$unused_highlight
-BUFFER='sudo -u otheruser ls /'
+# Tests three codepaths:
+# * -i  (no argument)
+# * -C3 (pasted argument)
+# * -u otheruser (non-pasted argument)
+BUFFER='sudo -C3 -u otheruser -i ls /; sudo ; sudo -u ;'
 
 expected_region_highlight=(
   "1 4 $ZSH_HIGHLIGHT_STYLES[precommand]" # sudo
-  "6 7 $ZSH_HIGHLIGHT_STYLES[single-hyphen-option]" # -u
-  "9 17 $ZSH_HIGHLIGHT_STYLES[default]" # otheruser
-  "19 20 $ZSH_HIGHLIGHT_STYLES[command]" # ls
-  "22 22 $ZSH_HIGHLIGHT_STYLES[path]" # /
+  "6 8 $ZSH_HIGHLIGHT_STYLES[single-hyphen-option]" # -C3
+  "10 11 $ZSH_HIGHLIGHT_STYLES[single-hyphen-option]" # -u
+  "13 21 $ZSH_HIGHLIGHT_STYLES[default]" # otheruser
+  "23 24 $ZSH_HIGHLIGHT_STYLES[single-hyphen-option]" # -i
+  "26 27 $ZSH_HIGHLIGHT_STYLES[command]" # ls
+  "29 29 $ZSH_HIGHLIGHT_STYLES[path]" # /
+  "37 37 $ZSH_HIGHLIGHT_STYLES[unknown-token]" # ;, error because empty command
+  "47 47 $ZSH_HIGHLIGHT_STYLES[unknown-token]" # ;, error because incomplete command
 )

highlighters/main/test-data/sudo-comment.zsh

@@ -0,0 +1,38 @@
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2015 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+ZSH_HIGHLIGHT_STYLES[single-hyphen-option]=$unused_highlight
+setopt interactive_comments
+BUFFER='sudo -u # comment'
+
+expected_region_highlight=(
+  "1 4 $ZSH_HIGHLIGHT_STYLES[precommand]" # sudo
+  "6 7 $ZSH_HIGHLIGHT_STYLES[single-hyphen-option]" # -u
+  "9 17 $ZSH_HIGHLIGHT_STYLES[unknown-token]" # "# comment" - error because argument missed
+)

highlighters/main/test-data/sudo-redirection.zsh

@@ -0,0 +1,46 @@
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2015 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER='sudo -u >/tmp otheruser ls; sudo ls; sudo -i ls'
+
+expected_region_highlight=(
+  "1 4 $ZSH_HIGHLIGHT_STYLES[precommand]" # sudo
+  "6 7 $ZSH_HIGHLIGHT_STYLES[single-hyphen-option]" # -u
+  "9 9 $ZSH_HIGHLIGHT_STYLES[redirection]" # >
+  "10 13 $ZSH_HIGHLIGHT_STYLES[path]" # /tmp
+  "15 23 $ZSH_HIGHLIGHT_STYLES[default]" # otheruser
+  "25 26 $ZSH_HIGHLIGHT_STYLES[command]" # ls
+  "27 27 $ZSH_HIGHLIGHT_STYLES[commandseparator]" # ;
+  "29 32 $ZSH_HIGHLIGHT_STYLES[precommand]" # sudo
+  "34 35 $ZSH_HIGHLIGHT_STYLES[command]" # ls
+  "36 36 $ZSH_HIGHLIGHT_STYLES[commandseparator]" # ;
+  "38 41 $ZSH_HIGHLIGHT_STYLES[precommand]" # sudo
+  "43 44 $ZSH_HIGHLIGHT_STYLES[single-hyphen-option]" # -i
+  "46 47 $ZSH_HIGHLIGHT_STYLES[command]" # ls
+)

highlighters/main/test-data/sudo-redirection2.zsh

@@ -0,0 +1,39 @@
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2015 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+BUFFER='sudo >/tmp -u otheruser ls'
+
+expected_region_highlight=(
+  "1 4 $ZSH_HIGHLIGHT_STYLES[precommand]" # sudo
+  "6 6 $ZSH_HIGHLIGHT_STYLES[redirection]" # >
+  "7 10 $ZSH_HIGHLIGHT_STYLES[path]" # /tmp
+  "12 13 $ZSH_HIGHLIGHT_STYLES[single-hyphen-option]" # -u
+  "15 23 $ZSH_HIGHLIGHT_STYLES[default]" # otheruser
+  "25 26 $ZSH_HIGHLIGHT_STYLES[command]" # ls
+)

highlighters/pattern/README.md

@@ -1,21 +0,0 @@
-zsh-syntax-highlighting / highlighters / pattern
-================================================
-
-This is the ***pattern*** highlighter, that highlights user defined patterns.
-
-
-How to activate it
-------------------
-To activate it, add it to `ZSH_HIGHLIGHT_HIGHLIGHTERS`:
-
-    ZSH_HIGHLIGHT_HIGHLIGHTERS=( [...] pattern)
-
-
-How to tweak it
----------------
-To use this highlighter, associate patterns with styles in the `ZSH_HIGHLIGHT_PATTERNS` array, for example in `~/.zshrc`:
-
-    # To have commands starting with `rm -rf` in red:
-    ZSH_HIGHLIGHT_PATTERNS+=('rm -rf *' 'fg=white,bold,bg=red')
-
-The syntax for declaring styles is [documented here](http://zsh.sourceforge.net/Doc/Release/Zsh-Line-Editor.html#SEC135).

highlighters/pattern/README.md

@@ -0,0 +1 @@
+../../docs/highlighters/pattern.md

highlighters/pattern/pattern-highlighter.zsh

@@ -52,6 +52,7 @@ _zsh_highlight_pattern_highlighter_loop()
   # This does *not* do its job syntactically, sorry.
   local buf="$1" pat="$2"
   local -a match mbegin mend
+  local MATCH; integer MBEGIN MEND
   if [[ "$buf" == (#b)(*)(${~pat})* ]]; then
     region_highlight+=("$((mbegin[2] - 1)) $mend[2] $ZSH_HIGHLIGHT_PATTERNS[$pat]")
     "$0" "$match[1]" "$pat"; return $?

highlighters/root/README.md

@@ -1,24 +0,0 @@
-zsh-syntax-highlighting / highlighters / root
-=================================================
-
-This is the ***root*** highlighter, that highlights the whole line if the current user is root.
-
-
-How to activate it
-------------------
-To activate it, add it to `ZSH_HIGHLIGHT_HIGHLIGHTERS`:
-
-    ZSH_HIGHLIGHT_HIGHLIGHTERS=( [...] root)
-
-
-How to tweak it
----------------
-This highlighter defines the following styles:
-
-* `root` - the style for the whole line if the current user is root.
-
-To override one of those styles, change its entry in `ZSH_HIGHLIGHT_STYLES`, for example in `~/.zshrc`:
-
-    ZSH_HIGHLIGHT_STYLES[root]='bg=red'
-
-The syntax for declaring styles is [documented here](http://zsh.sourceforge.net/Doc/Release/Zsh-Line-Editor.html#SEC135).

highlighters/root/README.md

@@ -0,0 +1 @@
+../../docs/highlighters/root.md

release.md

@@ -4,8 +4,12 @@
 - Confirm `make test` passes
   - check with multiple zsh versions
 - Update changelog.md
-- Update ./.version
-- `git tag $(<.version) && git push --tags`
-- `perl -pi -e 's/$/-dev/' ./.version`
+- Remove `-dev` suffix from `./.version`;
+  Commit that;
+  Tag it using `git tag $(<.version)`;
+  Increment `./.version` and restore the `-dev` suffix;
+  Commit that.
+- Push with `git push --tags`
 - Notify downstreams (OS packages)
   - anitya should autodetect the tag
+- Update /topic on IRC

tests/README.md

@@ -3,25 +3,53 @@ zsh-syntax-highlighting / tests
 
 Utility scripts for testing zsh-syntax-highlighting highlighters.
 
-The tests expect the highlighter directory to contain a `test-data` directory with test data files. See the [main highlighter](../highlighters/main/test-data) for examples.
+The tests harness expects the highlighter directory to contain a `test-data`
+directory with test data files.
+See the [main highlighter](../highlighters/main/test-data) for examples.
 
 Each test should define the array parameter `$expected_region_highlight`.
-The value of that parameter is a list of `"$i $j $style"` strings.
+The value of that parameter is a list of strings of the form  `"$i $j $style"`.
+or `"$i $j $style $todo"`.
 Each string specifies the highlighting that `$BUFFER[$i,$j]` should have;
 that is, `$i` and `$j` specify a range, 1-indexed, inclusive of both endpoints.
+If `$todo` exists, the test point is marked as TODO (the failure of that test
+point will not fail the test), and `$todo` is used as the explanation.
 
-_Note_: `$region_highlight` uses the same `"$i $j $style"` syntax but interprets the indexes differently.
+**Note**: `$region_highlight` uses the same `"$i $j $style"` syntax but
+interprets the indexes differently.
+
+**Isolation**: Each test is run in a separate subshell, so any variables,
+aliases, functions, etc., it defines will be visible to the tested code (that
+computes `$region_highlight`), but will not affect subsequent tests.  The
+current working directory of tests is set to a newly-created empty directory,
+which is automatically cleaned up after the test exits.
 
 
-highlighting test
+Highlighting test
 -----------------
-[`test-highlighting.zsh`](tests/test-highlighting.zsh) tests the correctness of the highlighting. Usage:
+
+[`test-highlighting.zsh`](tests/test-highlighting.zsh) tests the correctness of
+the highlighting. Usage:
 
     zsh test-highlighting.zsh <HIGHLIGHTER NAME>
 
+All tests may be run with
 
-performance test
+    make test
+
+which will run all highlighting tests and report results in [TAP format][TAP].
+
+[TAP]: http://testanything.org/
+
+
+Performance test
 ----------------
-[`test-perfs.zsh`](tests/test-perfs.zsh) measures the time spent doing the highlighting. Usage:
+
+[`test-perfs.zsh`](tests/test-perfs.zsh) measures the time spent doing the
+highlighting. Usage:
 
     zsh test-perfs.zsh <HIGHLIGHTER NAME>
+
+All tests may be run with
+
+    make perf

tests/tap-colorizer.zsh

@@ -0,0 +1,67 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2015 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+# This is a stdin-to-stdout filter that takes TAP output (such as 'make test')
+# on stdin and passes it, colorized, to stdout.
+
+emulate -LR zsh
+
+if [[ ! -t 1 ]] ; then
+  exec cat
+fi
+
+while read -r line;
+do
+  case $line in
+    # comment (filename header) or plan
+    (#* | <->..<->)
+      print -nP %F{blue}
+      ;;
+    # XPASS
+    (ok*# TODO*)
+      print -nP %F{red}
+      ;;
+    # XFAIL
+    (not ok*# TODO*)
+      print -nP %F{yellow}
+      ;;
+    # FAIL
+    (not ok*)
+      print -nP %F{red}
+      ;;
+    # PASS
+    (ok*)
+      print -nP %F{green}
+      ;;
+  esac
+  print -nr - "$line"
+  print -nP %f
+  echo "" # newline
+done

tests/test-highlighting.zsh

@@ -31,55 +31,48 @@
 
 # Check an highlighter was given as argument.
 [[ -n "$1" ]] || {
-  echo "You must provide the name of a valid highlighter as argument." >&2
-  exit 1
+  echo >&2 "Bail out! You must provide the name of a valid highlighter as argument."
+  exit 2
 }
 
 # Check the highlighter is valid.
 [[ -f ${0:h:h}/highlighters/$1/$1-highlighter.zsh ]] || {
-  echo "Could not find highlighter '$1'." >&2
-  exit 1
+  echo >&2 "Bail out! Could not find highlighter '$1'."
+  exit 2
 }
 
 # Check the highlighter has test data.
 [[ -d ${0:h:h}/highlighters/$1/test-data ]] || {
-  echo "Highlighter '$1' has no test data." >&2
-  exit 1
+  echo >&2 "Bail out! Highlighter '$1' has no test data."
+  exit 2
 }
 
-local -a errors highlight_zone
-local -A observed_result
-local -A save_ZSH_HIGHLIGHT_STYLES
-integer something_failed=0
-local unused_highlight='bg=red,underline' # a style unused by anything else, for tests to use
-
 # Load the main script.
 . ${0:h:h}/zsh-syntax-highlighting.zsh
 
 # Activate the highlighter.
 ZSH_HIGHLIGHT_HIGHLIGHTERS=($1)
 
-# Cache a pristine set of styles.
-save_ZSH_HIGHLIGHT_STYLES=( "${(@kv)ZSH_HIGHLIGHT_STYLES}" )
+# Runs a highlighting test
+# $1: data file
+run_test_internal() {
+  local -a highlight_zone
+  local unused_highlight='bg=red,underline' # a style unused by anything else, for tests to use
 
-# Process each test data file in test data directory.
-for data_file in ${0:h:h}/highlighters/$1/test-data/*.zsh; do
+  local tests_tempdir="$1"; shift
+  local srcdir="$PWD"
+  builtin cd -q -- "$tests_tempdir" || { echo >&2 "Bail out! cd failed: $?"; return 1 }
+
+  echo "# ${1:t:r}"
 
   # Load the data and prepare checking it.
-  PREBUFFER= BUFFER= ; expected_region_highlight=(); errors=()
-  echo -n "* ${data_file:t:r}: "
-  ZSH_HIGHLIGHT_STYLES=( "${(@kv)save_ZSH_HIGHLIGHT_STYLES}" )
-  . $data_file
+  PREBUFFER= BUFFER= ;
+  . "$srcdir"/"$1"
 
   # Check the data declares $PREBUFFER or $BUFFER.
-  if [[ ${#PREBUFFER} -eq 0 && ${#BUFFER} -eq 0 ]]; then
-    errors+=("Either 'PREBUFFER' or 'BUFFER' must be declared and non-blank")
-  else
-
+  [[ -z $PREBUFFER && -z $BUFFER ]] && { echo >&2 "Bail out! Either 'PREBUFFER' or 'BUFFER' must be declared and non-blank"; return 1; }
   # Check the data declares $expected_region_highlight.
-    if [[ ${#expected_region_highlight} -eq 0 ]]; then
-      errors+=("'expected_region_highlight' is not declared or empty.")
-    else
+  (( ${#expected_region_highlight} == 0 )) && { echo >&2 "Bail out! 'expected_region_highlight' is not declared or empty."; return 1; }
 
   # Process the data.
   region_highlight=()
@@ -87,8 +80,8 @@ for data_file in ${0:h:h}/highlighters/$1/test-data/*.zsh; do
 
   # Overlapping regions can be declared in region_highlight, so we first build an array of the
   # observed highlighting.
-      observed_result=()
-      for i in {1..${#region_highlight}}; do
+  local -A observed_result
+  for ((i=1; i<=${#region_highlight}; i++)); do
     highlight_zone=${(z)region_highlight[$i]}
     integer start=$highlight_zone[1] end=$highlight_zone[2]
     if (( start < end )) # region_highlight ranges are half-open
@@ -104,30 +97,41 @@ for data_file in ${0:h:h}/highlighters/$1/test-data/*.zsh; do
   done
 
   # Then we compare the observed result with the expected one.
-      for i in {1..${#expected_region_highlight}}; do
+  echo "1..${#expected_region_highlight}"
+  for ((i=1; i<=${#expected_region_highlight}; i++)); do
+    local todo=
     highlight_zone=${(z)expected_region_highlight[$i]}
+    [[ -n "$highlight_zone[4]" ]] && todo=" # TODO $highlight_zone[4]"
     for j in {$highlight_zone[1]..$highlight_zone[2]}; do
       if [[ "$observed_result[$j]" != "$highlight_zone[3]" ]]; then
-            errors+=("'$BUFFER[$highlight_zone[1],$highlight_zone[2]]' [$highlight_zone[1],$highlight_zone[2]]: expected '$highlight_zone[3]', observed '$observed_result[$j]'.")
-            break
+        echo "not ok $i '$BUFFER[$highlight_zone[1],$highlight_zone[2]]' [$highlight_zone[1],$highlight_zone[2]]: expected '$highlight_zone[3]', observed '$observed_result[$j]'.$todo"
+        continue 2
       fi
     done
+    echo "ok $i$todo"
   done
+}
 
-    fi
-  fi
+run_test() {
+  # Do not combine the declaration and initialization: «local x="$(false)"» does not set $?.
+  local __tests_tempdir
+  __tests_tempdir="$(mktemp -d)" && [[ -d $__tests_tempdir ]] || {
+    echo >&2 "Bail out! mktemp failed"; return 1
+  }
+  typeset -r __tests_tempdir # don't allow tests to override the variable that we will 'rm -rf' later on
 
-  # Format result/errors.
-  if [[ ${#errors} -eq 0 ]]; then
-    echo "OK"
-  else
-    echo "KO"
-    (( something_failed=1 ))
-    for error in $errors; do
-      echo "   - $error"
-    done
-  fi
+  {
+    (run_test_internal "$__tests_tempdir" "$@")
+  } always {
+    rm -rf -- "$__tests_tempdir"
+  }
+}
 
+# Process each test data file in test data directory.
+integer something_failed=0
+for data_file in ${0:h:h}/highlighters/$1/test-data/*.zsh; do
+  run_test "$data_file" | tee >(${0:A:h}/tap-colorizer.zsh) | grep -v '^not ok.*# TODO' | grep -q '^not ok\|^ok.*# TODO' && (( something_failed=1 ))
+  (( $pipestatus[1] )) && exit 2
 done
 
 exit $something_failed

tests/test-perfs.zsh

@@ -1,6 +1,6 @@
 #!/usr/bin/env zsh
 # -------------------------------------------------------------------------------------------------
-# Copyright (c) 2010-2011 zsh-syntax-highlighting contributors
+# Copyright (c) 2010-2015 zsh-syntax-highlighting contributors
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification, are permitted
@@ -31,20 +31,20 @@
 
 # Check an highlighter was given as argument.
 [[ -n "$1" ]] || {
-  echo "You must provide the name of a valid highlighter as argument." >&2
-  exit 1
+  echo >&2 "Bail out! You must provide the name of a valid highlighter as argument."
+  exit 2
 }
 
 # Check the highlighter is valid.
 [[ -f ${0:h:h}/highlighters/$1/$1-highlighter.zsh ]] || {
-  echo "Could not find highlighter '$1'." >&2
-  exit 1
+  echo >&2 "Bail out! Could not find highlighter '$1'."
+  exit 2
 }
 
 # Check the highlighter has test data.
 [[ -d ${0:h:h}/highlighters/$1/test-data ]] || {
-  echo "Highlighter '$1' has no test data." >&2
-  exit 1
+  echo >&2 "Bail out! Highlighter '$1' has no test data."
+  exit 2
 }
 
 # Load the main script.
@@ -53,23 +53,49 @@
 # Activate the highlighter.
 ZSH_HIGHLIGHT_HIGHLIGHTERS=($1)
 
-# Process each test data file in test data directory.
-for data_file in ${0:h:h}/highlighters/$1/test-data/*; do
+# Runs a highlighting test
+# $1: data file
+run_test_internal() {
+  local -a highlight_zone
+  local unused_highlight='bg=red,underline' # a style unused by anything else, for tests to use
+
+  local tests_tempdir="$1"; shift
+  local srcdir="$PWD"
+  builtin cd -q -- "$tests_tempdir" || { echo >&2 "Bail out! cd failed: $?"; return 1 }
+
+  echo -n "# ${1:t:r}: "
 
   # Load the data and prepare checking it.
-  BUFFER=
-  echo -n "* ${data_file:t:r}: "
-  . $data_file
+  PREBUFFER= BUFFER= ;
+  . "$srcdir"/"$1"
 
-  # Check the data declares $BUFFER.
-  if [[ ${#BUFFER} -eq 0 ]]; then
-    echo "KO\n   - 'BUFFER' is not declared or blank."
-  else
+  # Check the data declares $PREBUFFER or $BUFFER.
+  [[ -z $PREBUFFER && -z $BUFFER ]] && { echo >&2 "Bail out! Either 'PREBUFFER' or 'BUFFER' must be declared and non-blank"; return 1; }
 
   # Measure the time taken by _zsh_highlight.
   TIMEFMT="%*Es"
-    time ( BUFFER="$BUFFER" && _zsh_highlight)
+  time (BUFFER="$BUFFER" && _zsh_highlight)
+}
 
-  fi
+run_test() {
+  # Do not combine the declaration and initialization: «local x="$(false)"» does not set $?.
+  local __tests_tempdir
+  __tests_tempdir="$(mktemp -d)" && [[ -d $__tests_tempdir ]] || {
+    echo >&2 "Bail out! mktemp failed"; return 1
+  }
+  typeset -r __tests_tempdir # don't allow tests to override the variable that we will 'rm -rf' later on
 
+  {
+    (run_test_internal "$__tests_tempdir" "$@")
+  } always {
+    rm -rf -- "$__tests_tempdir"
+  }
+}
+
+# Process each test data file in test data directory.
+for data_file in ${0:h:h}/highlighters/$1/test-data/*.zsh; do
+  run_test "$data_file"
+  (( $pipestatus[1] )) && exit 2
 done
+
+exit 0

zsh-syntax-highlighting.zsh

@@ -30,19 +30,19 @@
 
 if [[ -o function_argzero ]]; then
   # $0 is reliable
-  ZSH_HIGHLIGHT_VERSION=$(<"$0:h"/.version)
-  ZSH_HIGHLIGHT_REVISION=$(<"$0:h"/.revision-hash)
+  ZSH_HIGHLIGHT_VERSION=$(<"${0:A:h}"/.version)
+  ZSH_HIGHLIGHT_REVISION=$(<"${0:A:h}"/.revision-hash)
   if [[ $ZSH_HIGHLIGHT_REVISION == \$Format:* ]]; then
     # When running from a source tree without 'make install', $ZSH_HIGHLIGHT_REVISION
     # would be set to '$Format:%H$' literally.  That's an invalid value, and obtaining
     # the valid value (via `git rev-parse HEAD`, as Makefile does) might be costly, so:
-    unset ZSH_HIGHLIGHT_REVISION
+    ZSH_HIGHLIGHT_REVISION=HEAD
   fi
 else
   # $0 is unreliable, so the call to _zsh_highlight_load_highlighters will fail.
   # TODO: If 'zmodload zsh/parameter' is available, ${funcsourcetrace[1]%:*} might serve as a substitute?
   # TODO: also check POSIX_ARGZERO, but not it's not available in older zsh
-  echo "zsh-syntax-highlighting: error: not compatible with FUNCTION_ARGZERO" >&2
+  echo "zsh-syntax-highlighting: error: not compatible with NO_FUNCTION_ARGZERO" >&2
   return 1
 fi
 
@@ -63,6 +63,7 @@ _zsh_highlight()
   local ret=$?
 
   setopt localoptions warncreateglobal
+  setopt localoptions noksharrays
   local REPLY # don't leak $REPLY into global scope
 
   # Do not highlight if there are more than 300 chars in the buffer. It's most
@@ -178,9 +179,19 @@ _zsh_highlight_cursor_moved()
 # Setup functions
 # -------------------------------------------------------------------------------------------------
 
+# Helper for _zsh_highlight_bind_widgets
+# $1 is name of widget to call
+_zsh_highlight_call_widget()
+{
+  builtin zle "$@" && 
+  _zsh_highlight
+}
+
 # Rebind all ZLE widgets to make them invoke _zsh_highlights.
 _zsh_highlight_bind_widgets()
 {
+  setopt localoptions noksharrays
+
   # Load ZSH module zsh/zleparameter, needed to override user defined widgets.
   zmodload zsh/zleparameter 2>/dev/null || {
     echo 'zsh-syntax-highlighting: failed loading zsh/zleparameter.' >&2
@@ -197,16 +208,16 @@ _zsh_highlight_bind_widgets()
 
       # User defined widget: override and rebind old one with prefix "orig-".
       user:*) eval "zle -N orig-$cur_widget ${widgets[$cur_widget]#*:}; \
-                    _zsh_highlight_widget_$cur_widget() { builtin zle orig-$cur_widget -- \"\$@\" && _zsh_highlight }; \
+                    _zsh_highlight_widget_$cur_widget() { _zsh_highlight_call_widget orig-$cur_widget -- \"\$@\" }; \
                     zle -N $cur_widget _zsh_highlight_widget_$cur_widget";;
 
       # Completion widget: override and rebind old one with prefix "orig-".
       completion:*) eval "zle -C orig-$cur_widget ${${widgets[$cur_widget]#*:}/:/ }; \
-                          _zsh_highlight_widget_$cur_widget() { builtin zle orig-$cur_widget -- \"\$@\" && _zsh_highlight }; \
+                          _zsh_highlight_widget_$cur_widget() { _zsh_highlight_call_widget orig-$cur_widget -- \"\$@\" }; \
                           zle -N $cur_widget _zsh_highlight_widget_$cur_widget";;
 
       # Builtin widget: override and make it call the builtin ".widget".
-      builtin) eval "_zsh_highlight_widget_$cur_widget() { builtin zle .$cur_widget -- \"\$@\" && _zsh_highlight }; \
+      builtin) eval "_zsh_highlight_widget_$cur_widget() { _zsh_highlight_call_widget .$cur_widget -- \"\$@\" }; \
                      zle -N $cur_widget _zsh_highlight_widget_$cur_widget";;
 
       # Default: unhandled case.
@@ -221,6 +232,8 @@ _zsh_highlight_bind_widgets()
 #   1) Path to the highlighters directory.
 _zsh_highlight_load_highlighters()
 {
+  setopt localoptions noksharrays
+
   # Check the directory exists.
   [[ -d "$1" ]] || {
     echo "zsh-syntax-highlighting: highlighters directory '$1' not found." >&2