Merge pull request #1424 from sanjaymsh/ppc64le

Travis-ci: added support for ppc64le

.gitignore

@@ -1,4 +1,5 @@
 *.dSYM
+*.gcda
 *.o
 *.plist
 .deps

.travis.yml

@@ -1,27 +1,50 @@
 language: c
+dist: xenial
+sudo: false
 
 branches:
   only:
     - master
-
-notifications:
-  email:
-    - geoff@greer.fm
+    - ppc64le
+arch:
+  - amd64
+  - ppc64le
 
 compiler:
   - clang
   - gcc
 
-before_install:
-  - sudo add-apt-repository -y 'ppa:ubuntu-toolchain-r/test'
-  - sudo add-apt-repository -y 'deb http://llvm.org/apt/precise/ llvm-toolchain-precise-3.6 main'
-  - curl http://llvm.org/apt/llvm-snapshot.gpg.key | sudo apt-key add -
-  - sudo apt-get update -q
-  - sudo apt-get install -q -y automake pkg-config libpcre3-dev zlib1g-dev liblzma-dev
-  - sudo apt-get install -y clang-format-3.6
+addons:
+  apt:
+    sources:
+      - ubuntu-toolchain-r-test
+    packages:
+      - automake
+      - liblzma-dev
+      - libpcre3-dev
+      - pkg-config
+      - zlib1g-dev
 
-before_script:
-  - sudo pip install cram
+env:
+  global:
+    - LLVM_VERSION=6.0.1
+    - LLVM_PATH=$HOME/clang+llvm
+    - CLANG_FORMAT=$LLVM_PATH/bin/clang-format
+
+before_install:
+  - wget http://llvm.org/releases/$LLVM_VERSION/clang+llvm-$LLVM_VERSION-x86_64-linux-gnu-ubuntu-16.04.tar.xz -O $LLVM_PATH.tar.xz
+  - mkdir $LLVM_PATH
+  - tar xf $LLVM_PATH.tar.xz -C $LLVM_PATH --strip-components=1
+  - export PATH=$HOME/.local/bin:$PATH
+
+install:
+  - pip install --user cram
 
 script:
   - ./build.sh && make test
+
+notifications:
+  irc: 'chat.freenode.net#ag'
+  on_success: change
+  on_failure: always
+  use_notice: true

CONTRIBUTING.md

@@ -14,3 +14,13 @@ The test suite uses [Cram](https://bitheap.org/cram/). You'll need to build ag
 first, and then you can run the suite from the root of the repository :
 
     make test
+
+### Adding filetypes
+
+Ag can search files which belong to a certain class for example `ag --html test` 
+searches all files with the extension defined in [lang.c](src/lang.c).
+
+If you want to add a new file 'class' to ag please modify [lang.c](src/lang.c) and [list_file_types.t](tests/list_file_types.t).
+
+`lang.c` adds the functionality and `list_file_types.t` adds the test case. 
+Without adding a test case the test __will__ fail.

Makefile.am

@@ -1,16 +1,21 @@
 ACLOCAL_AMFLAGS = ${ACLOCAL_FLAGS}
 
 bin_PROGRAMS = ag
-ag_SOURCES = src/ignore.c src/ignore.h src/log.c src/log.h src/options.c src/options.h src/print.c src/print_w32.c src/print.h src/scandir.c src/scandir.h src/search.c src/search.h src/lang.c src/lang.h src/util.c src/util.h src/decompress.c src/decompress.h src/uthash.h src/main.c
+ag_SOURCES = src/ignore.c src/ignore.h src/log.c src/log.h src/options.c src/options.h src/print.c src/print_w32.c src/print.h src/scandir.c src/scandir.h src/search.c src/search.h src/lang.c src/lang.h src/util.c src/util.h src/decompress.c src/decompress.h src/uthash.h src/main.c src/zfile.c
 ag_LDADD = ${PCRE_LIBS} ${LZMA_LIBS} ${ZLIB_LIBS} $(PTHREAD_LIBS)
 
 dist_man_MANS = doc/ag.1
 
 bashcompdir = $(pkgdatadir)/completions
 dist_bashcomp_DATA = ag.bashcomp.sh
+zshcompdir = $(datadir)/zsh/site-functions
+dist_zshcomp_DATA = _the_silver_searcher
 
 EXTRA_DIST = Makefile.w32 LICENSE NOTICE the_silver_searcher.spec README.md
 
+all:
+	@$(MAKE) ag -r
+
 test: ag
 	cram -v tests/*.t
 if HAS_CLANG_FORMAT
@@ -22,4 +27,7 @@ endif
 test_big: ag
 	cram -v tests/big/*.t
 
-.PHONY : all test clean
+test_fail: ag
+	cram -v tests/fail/*.t
+
+.PHONY : all clean test test_big test_fail

Makefile.w32

@@ -1,7 +1,8 @@
-GREP=grep
-VERSION:=$(shell $(GREP) -Po "(?<=\[)([0-9.]+.[0-9]+.[0-9]+)(?=\])" configure.ac)
+SED=sed
+VERSION:=$(shell "$(SED)" -n "s/[^[]*\[\([0-9]\+\.[0-9]\+\.[0-9]\+\)\],/\1/p" configure.ac)
 
 CC=gcc
+RM=/bin/rm
 
 SRCS = \
 	src/decompress.c \
@@ -23,11 +24,12 @@ TARGET = ag.exe
 
 all : $(TARGET)
 
-$(TARGET) : $(OBJS)
+# depend on configure.ac to account for version changes
+$(TARGET) : $(OBJS) configure.ac
 	$(CC) -o $@ $(OBJS) $(LIBS)
 
 .c.o :
 	$(CC) -c $(CFLAGS) -Isrc $< -o $@
 
 clean :
-	rm -f src/*.o $(TARGET)
+	$(RM) -f src/*.o $(TARGET)

NOTICE

@@ -1,2 +1,2 @@
 The Silver Searcher
-Copyright 2011-2014 Geoff Greer
+Copyright 2011-2016 Geoff Greer

README.md

@@ -6,16 +6,16 @@ A code searching tool similar to `ack`, with a focus on speed.
 
 [![Floobits Status](https://floobits.com/ggreer/ag.svg)](https://floobits.com/ggreer/ag/redirect)
 
-[![#ag on Freenode](http://img.shields.io/Freenode/%23ag.png)](https://webchat.freenode.net/?channels=ag)
+[![#ag on Freenode](https://img.shields.io/badge/Freenode-%23ag-brightgreen.svg)](https://webchat.freenode.net/?channels=ag)
 
-Do you know C? I invite you to pair with me to [help me get to Ag 1.0](http://geoff.greer.fm/2014/10/13/help-me-get-to-ag-10/).
+Do you know C? Want to improve ag? [I invite you to pair with me](http://geoff.greer.fm/2014/10/13/help-me-get-to-ag-10/).
 
 
 ## What's so great about Ag?
 
 * It is an order of magnitude faster than `ack`.
 * It ignores file patterns from your `.gitignore` and `.hgignore`.
-* If there are files in your source repo you don't want to search, just add their patterns to a `.agignore` file. (\*cough\* extern \*cough\*)
+* If there are files in your source repo you don't want to search, just add their patterns to a `.ignore` file. (\*cough\* `*.min.js` \*cough\*)
 * The command name is 33% shorter than `ack`, and all keys are on the home row!
 
 Ag is quite stable now. Most changes are new features, minor bug fixes, or performance improvements. It's much faster than Ack in my benchmarks:
@@ -24,7 +24,7 @@ Ag is quite stable now. Most changes are new features, minor bug fixes, or perfo
 
     ag test_blah ~/code/  4.67s user 4.58s system 286% cpu 3.227 total
 
-Ack and Ag found the same results, but Ag was 34x faster (3.2 seconds vs 110 seconds). My `~/code` directory is about 8GB. Thanks to git/hg/svn-ignore, Ag only searched 700MB of that.
+Ack and Ag found the same results, but Ag was 34x faster (3.2 seconds vs 110 seconds). My `~/code` directory is about 8GB. Thanks to git/hg/ignore, Ag only searched 700MB of that.
 
 There are also [graphs of performance across releases](http://geoff.greer.fm/ag/speed/).
 
@@ -42,7 +42,7 @@ I've written several blog posts showing how I've improved performance. These inc
 
 ## Installing
 
-### OS X
+### macOS
 
     brew install the_silver_searcher
 
@@ -56,16 +56,18 @@ or
 * Ubuntu >= 13.10 (Saucy) or Debian >= 8 (Jessie)
 
         apt-get install silversearcher-ag
-* Fedora 19+
+* Fedora 21 and lower
 
         yum install the_silver_searcher
+* Fedora 22+
+
+        dnf install the_silver_searcher
 * RHEL7+
 
-        rpm -Uvh http://download.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-5.noarch.rpm
-        yum install the_silver_searcher
+        yum install epel-release.noarch the_silver_searcher
 * Gentoo
 
-        emerge the_silver_searcher
+        emerge -a sys-apps/the_silver_searcher
 * Arch
 
         pacman -S the_silver_searcher
@@ -74,6 +76,20 @@ or
 
         sbopkg -i the_silver_searcher
 
+* openSUSE
+
+        zypper install the_silver_searcher
+
+* CentOS
+
+        yum install the_silver_searcher
+
+* NixOS/Nix/Nixpkgs
+
+        nix-env -iA silver-searcher
+
+* SUSE Linux Enterprise: Follow [these simple instructions](https://software.opensuse.org/download.html?project=utilities&package=the_silver_searcher).
+
 
 ### BSD
 
@@ -84,14 +100,42 @@ or
 
         pkg_add the_silver_searcher
 
+### Windows
 
+* Win32/64
+
+  Unofficial daily builds are [available](https://github.com/k-takata/the_silver_searcher-win32).
+  
+* winget
+
+        winget install "The Silver Searcher"
+  
+  Notes:
+  - This installs a [release](https://github.com/JFLarvoire/the_silver_searcher/releases) of ag.exe optimized for Windows.
+  - winget is intended to become the default package manager client for Windows.  
+    As of June 2020, it's still in beta, and can be installed using instructions [there](https://github.com/microsoft/winget-cli).
+  - The setup script in the Ag's winget package installs ag.exe in the first directory that matches one of these criteria:
+     1. Over a previous instance of ag.exe *from the same [origin](https://github.com/JFLarvoire/the_silver_searcher)* found in the PATH
+     2. In the directory defined in environment variable bindir_%PROCESSOR_ARCHITECTURE%
+     3. In the directory defined in environment variable bindir
+     4. In the directory defined in environment variable windir
+  
+* Chocolatey
+
+        choco install ag
+* MSYS2
+
+        pacman -S mingw-w64-{i686,x86_64}-ag
+* Cygwin
+
+  Run the relevant [`setup-*.exe`](https://cygwin.com/install.html), and select "the\_silver\_searcher" in the "Utils" category.
 
 ## Building from source
 
 ### Building master
 
 1. Install dependencies (Automake, pkg-config, PCRE, LZMA):
-    * OS X:
+    * macOS:
 
             brew install automake pkg-config pcre xz
         or
@@ -106,15 +150,19 @@ or
     * CentOS:
 
             yum -y groupinstall "Development Tools"
-            yum -y install pcre-devel xz-devel
+            yum -y install pcre-devel xz-devel zlib-devel
+    * openSUSE:
+
+            zypper source-install --build-deps-only the_silver_searcher
+
     * Windows: It's complicated. See [this wiki page](https://github.com/ggreer/the_silver_searcher/wiki/Windows).
 2. Run the build script (which just runs aclocal, automake, etc):
 
         ./build.sh
 
-  On Windows:
+   On Windows (inside an msys/MinGW shell):
 
-        mingw32-make -f Makefile.w32
+        make -f Makefile.w32
 3. Make install:
 
         sudo make install
@@ -137,7 +185,7 @@ You may need to use `sudo` or run as root for the make install.
 
 ### Vim
 
-You can use Ag with [ack.vim][] by adding the following line to your `.vimrc`:
+You can use Ag with [ack.vim](https://github.com/mileszs/ack.vim) by adding the following line to your `.vimrc`:
 
     let g:ackprg = 'ag --nogroup --nocolor --column'
 
@@ -147,13 +195,9 @@ or:
 
 Which has the same effect but will report every match on the line.
 
-There's also a fork of ack.vim tailored for use with Ag: [ag.vim][]
-[ack.vim]: https://github.com/mileszs/ack.vim
-[ag.vim]: https://github.com/rking/ag.vim
-
 ### Emacs
 
-You can use [ag.el][] as an Emacs fronted to Ag. See also: [helm-ag].
+You can use [ag.el][] as an Emacs front-end to Ag. See also: [helm-ag].
 
 [ag.el]: https://github.com/Wilfred/ag.el
 [helm-ag]: https://github.com/syohex/emacs-helm-ag
@@ -164,10 +208,10 @@ TextMate users can use Ag with [my fork](https://github.com/ggreer/AckMate) of t
 
 ## Other stuff you might like
 
-* [Ack](https://github.com/petdance/ack2) - Better than grep. Without Ack, Ag would not exist.
-* [AckMate](https://github.com/protocool/AckMate) - An ack-powered replacement for TextMate's slow built-in search.
+* [Ack](https://github.com/petdance/ack3) - Better than grep. Without Ack, Ag would not exist.
 * [ack.vim](https://github.com/mileszs/ack.vim)
-* [ag.vim]( https://github.com/rking/ag.vim)
 * [Exuberant Ctags](http://ctags.sourceforge.net/) - Faster than Ag, but it builds an index beforehand. Good for *really* big codebases.
 * [Git-grep](http://git-scm.com/docs/git-grep) - As fast as Ag but only works on git repos.
+* [fzf](https://github.com/junegunn/fzf) - A command-line fuzzy finder 
+* [ripgrep](https://github.com/BurntSushi/ripgrep)
 * [Sack](https://github.com/sampson-chen/sack) - A utility that wraps Ack and Ag. It removes a lot of repetition from searching and opening matching files.

_the_silver_searcher

@@ -0,0 +1,86 @@
+#compdef ag
+
+# Completion function for zsh
+
+local ret=1
+local -a args expl
+
+# Intentionally avoided many possible mutual exlusions because it is
+# likely that earlier options come from an alias. In line with this
+# the following conditionally adds options that assert defaults.
+[[ -n $words[(r)(-[is]|--ignore-case|--case-sensitive)] ]] && args+=(
+  '(-S --smart-case -s -s --ignore-case --case-sensitive)'{-S,--smart-case}'[insensitive match unless pattern includes uppercase]'
+)
+[[ -n $words[(r)--nobreak] ]] && args+=(
+  "(--nobreak)--break[print newlines between matches in different files]"
+)
+[[ -n $words[(r)--nogroup] ]] && args+=(
+    "(--nogroup)--group[don't repeat filename for each match line]"
+)
+
+_tags normal-options file-types
+while _tags; do
+  _requested normal-options && _arguments -S -s $args \
+    '--ackmate[print results in AckMate-parseable format]' \
+    '(--after -A)'{--after=-,-A+}'[specify lines of trailing context]::lines [2]' \
+    '(--before -B)'{--before=-,-B+}'[specify lines of leading context]::lines [2]' \
+    "--nobreak[don't print newlines between matches in different files]" \
+    '(--count -c)'{--count,-c}'[only print a count of matching lines]' \
+    '--color[enable color highlighting of output]' \
+    '(--color-line-number --color-match --color-path)--nocolor[disable color highlighting of output]' \
+    '--color-line-number=[specify color for line numbers]:color [1;33]' \
+    '--color-match=[specify color for result match numbers]:color [30;43]' \
+    '--color-path=[specify color for path names]:color [1;32]' \
+    '--column[print column numbers in results]' \
+    '(--context -C)'{--context=-,-C+}'[specify lines of context]::lines' \
+    '(--debug -D)'{--debug,-D}'[output debug information]' \
+    '--depth=[specify directory levels to descend when searching]:levels [25]' \
+    '(--noheading)--nofilename[suppress printing of filenames]' \
+    '(-f --follow)'{-f,--follow}'[follow symlinks]' \
+    '(-F --fixed-strings --literal -Q)'{--fixed-strings,-F,--literal,-Q}'[use literal strings]' \
+    '--nogroup[repeat filename for each match line]' \
+    '(1 -G --file-search-regex)-g+[print filenames matching a pattern]:regex' \
+    '(-G --file-search-regex)'{-G+,--file-search-regex=}'[limit search to filenames matching pattern]:regex' \
+    '(-H --heading --noheading)'{-H,--heading}'[print filename with each match]' \
+    '(-H --heading --noheading --nofilename)--noheading[suppress printing of filenames]' \
+    '--hidden[search hidden files (obeying .*ignore files)]' \
+    {--ignore=,--ignore-dir=}'[ignore files/directories matching pattern]:regex' \
+    '(-i --ignore-case)'{-i,--ignore-case}'[match case-insensitively]' \
+    '(-l --files-with-matches)'{-l,--files-with-matches}"[output matching files' names only]" \
+    '(-L --files-without-matches)'{-L,--files-without-matches}"[output non-matching files' names only]" \
+    '(--max-count -m)'{--max-count=,-m+}'[stop after specified no of matches in each file]:max number of matches' \
+    '--numbers[prefix output with line numbers, even for streams]' \
+    '--nonumbers[suppress printing of line numbers]' \
+    '(--only-matching -o)'{--only-matching,-o}'[show only matching part of line]' \
+    '(-p --path-to-ignore)'{-p+,--path-to-ignore=}'[use specified .ignore file]:file:_files' \
+    '--print-long-lines[print matches on very long lines]' \
+    "--passthrough[when searching a stream, print all lines even if they don't match]" \
+    '(-s --case-sensitive)'{-s,--case-sensitive}'[match case]' \
+    '--silent[suppress all log messages, including errors]' \
+    '(--stats-only)--stats[print stats (files scanned, time taken, etc.)]' \
+    '(--stats)--stats-only[print stats and nothing else]' \
+    '(-U --skip-vcs-ignores)'{-U,--skip-vcs-ignores}'[ignore VCS files (stil obey .ignore)]' \
+    '(-v --invert-match)'{-v,--invert-match}'[select non-matching lines]' \
+    '--vimgrep[output results like vim :vimgrep /pattern/g would]' \
+    '(-w --word-regexp)'{-w,--word-regexp}'[force pattern to match only whole words]' \
+    '(-z --search-zip)'{-z,--search-zip}'[search contents of compressed files]' \
+    '(-0 --null)'{-0,--null}'[separate filenames with null]' \
+    ': :_guard "^-*" pattern' \
+    '*:file:_files' \
+    '(- :)--list-file-types[list supported file types]' \
+    '(- :)'{-h,--help}'[display help information]' \
+    '(- :)'{-V,--version}'[display version information]' \
+    - '(ignores)' \
+    '(-a --all-types)'{-a,--all-types}'[search all files]' \
+    '--search-binary[search binary files for matches]' \
+    {-t,--all-text}'[search all text files (not including hidden files)]' \
+    {-u,--unrestricted}'[search all files]' && ret=0
+
+ _requested file-types && { ! zstyle -T ":completion:${curcontext}:options" prefix-needed ||
+     [[ -prefix - ]] } && _all_labels file-types expl 'file type' \
+     compadd - ${(M)$(_call_program file-types $words[1] --list-file-types):#--*} && ret=0
+
+  (( ret )) || break
+done
+
+return ret

ag.bashcomp.sh

@@ -67,7 +67,7 @@ _ag() {
     --parallel
     --passthrough
     --passthru
-    --path-to-agignore
+    --path-to-ignore
     --print-long-lines
     --print0
     --recurse
@@ -106,7 +106,7 @@ _ag() {
     --ignore-dir) # directory completion
               _filedir -d
               return 0;;
-    --path-to-agignore) # file completion
+    --path-to-ignore) # file completion
               _filedir
               return 0;;
     --pager) # command completion
@@ -129,4 +129,7 @@ _ag() {
           return 0;;
   esac
 } &&
+
+# shellcheck disable=SC2086
+# shellcheck disable=SC2154,SC2086
 complete -F _ag ${nospace} ag

autogen.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+set -e
+cd "$(dirname "$0")"
+
+AC_SEARCH_OPTS=""
+# For those of us with pkg-config and other tools in /usr/local
+PATH=$PATH:/usr/local/bin
+
+# This is to make life easier for people who installed pkg-config in /usr/local
+# but have autoconf/make/etc in /usr/. AKA most mac users
+if [ -d "/usr/local/share/aclocal" ]
+then
+    AC_SEARCH_OPTS="-I /usr/local/share/aclocal"
+fi
+
+# shellcheck disable=2086
+aclocal $AC_SEARCH_OPTS
+autoconf
+autoheader
+automake --add-missing

build.sh

@@ -1,21 +1,8 @@
 #!/bin/sh
 
-cd `dirname $0`
+set -e
+cd "$(dirname "$0")"
 
-AC_SEARCH_OPTS=""
-# For those of us with pkg-config and other tools in /usr/local
-PATH=$PATH:/usr/local/bin
-
-# This is to make life easier for people who installed pkg-config in /usr/local
-# but have autoconf/make/etc in /usr/. AKA most mac users
-if [ -d "/usr/local/share/aclocal" ]
-then
-    AC_SEARCH_OPTS="-I /usr/local/share/aclocal"
-fi
-
-aclocal $AC_SEARCH_OPTS && \
-autoconf && \
-autoheader && \
-automake --add-missing && \
-./configure "$@" && \
+./autogen.sh
+./configure "$@"
 make -j4

configure.ac

@@ -1,6 +1,6 @@
 AC_INIT(
     [the_silver_searcher],
-    [0.30.0],
+    [2.2.0],
     [https://github.com/ggreer/the_silver_searcher/issues],
     [the_silver_searcher],
     [https://github.com/ggreer/the_silver_searcher])
@@ -10,6 +10,7 @@ AM_INIT_AUTOMAKE([no-define foreign subdir-objects])
 AC_PROG_CC
 AM_PROG_CC_C_O
 AC_PREREQ([2.59])
+AC_PROG_GREP
 
 m4_ifdef(
     [AM_SILENT_RULES],
@@ -24,7 +25,12 @@ AX_PTHREAD(
 )
 
 # Run CFLAGS="-pg" ./configure if you want debug symbols
-CFLAGS="$CFLAGS $PTHREAD_CFLAGS $PCRE_CFLAGS -Wall -Wextra -Wformat=2 -Wno-format-nonliteral -Wshadow -Wpointer-arith -Wcast-qual -Wmissing-prototypes -Wno-missing-braces -std=gnu89 -D_GNU_SOURCE -O2"
+if ! echo "$CFLAGS" | "$GREP" '\(^\|[[[:space:]]]\)-O' > /dev/null; then
+    CFLAGS="$CFLAGS -O2"
+fi
+
+CFLAGS="$CFLAGS $PTHREAD_CFLAGS $PCRE_CFLAGS -Wall -Wextra -Wformat=2 -Wno-format-nonliteral -Wshadow"
+CFLAGS="$CFLAGS -Wpointer-arith -Wcast-qual -Wmissing-prototypes -Wno-missing-braces -std=gnu89 -D_GNU_SOURCE"
 LDFLAGS="$LDFLAGS"
 
 case $host in
@@ -53,18 +59,19 @@ AS_IF([test "x$enable_lzma" != "xno"], [
 AC_CHECK_DECL([PCRE_CONFIG_JIT], [AC_DEFINE([USE_PCRE_JIT], [], [Use PCRE JIT])], [], [#include <pcre.h>])
 
 AC_CHECK_DECL([CPU_ZERO, CPU_SET], [AC_DEFINE([USE_CPU_SET], [], [Use CPU_SET macros])] , [], [#include <sched.h>])
+AC_CHECK_HEADERS([sys/cpuset.h err.h])
 
 AC_CHECK_MEMBER([struct dirent.d_type], [AC_DEFINE([HAVE_DIRENT_DTYPE], [], [Have dirent struct member d_type])], [], [[#include <dirent.h>]])
 AC_CHECK_MEMBER([struct dirent.d_namlen], [AC_DEFINE([HAVE_DIRENT_DNAMLEN], [], [Have dirent struct member d_namlen])], [], [[#include <dirent.h>]])
 
-AC_CHECK_FUNCS(fgetln getline realpath strlcpy strndup vasprintf madvise posix_fadvise pthread_setaffinity_np)
+AC_CHECK_FUNCS(fgetln fopencookie getline realpath strlcpy strndup vasprintf madvise posix_fadvise pthread_setaffinity_np pledge)
 
 AC_CONFIG_FILES([Makefile the_silver_searcher.spec])
 AC_CONFIG_HEADERS([src/config.h])
 
 AC_CHECK_PROGS(
     [CLANG_FORMAT],
-    [clang-format-3.7 clang-format-3.6 clang-format],
+    [clang-format-3.8 clang-format-3.7 clang-format-3.6 clang-format],
     [no]
 )
 AM_CONDITIONAL([HAS_CLANG_FORMAT], [test x$CLANG_FORMAT != xno])

doc/ag.1

@@ -1,13 +1,13 @@
 .\" generated with Ronn/v0.7.3
 .\" http://github.com/rtomayko/ronn/tree/0.7.3
 .
-.TH "AG" "1" "February 2015" "" ""
+.TH "AG" "1" "December 2016" "" ""
 .
 .SH "NAME"
 \fBag\fR \- The Silver Searcher\. Like ack, but faster\.
 .
 .SH "SYNOPSIS"
-\fBag\fR [\fIfile\-type\fR] [\fIoptions\fR] PATTERN [PATH]
+\fBag\fR [\fIoptions\fR] \fIpattern\fR [\fIpath \.\.\.\fR]
 .
 .SH "DESCRIPTION"
 Recursively search for PATTERN in PATH\. Like grep or ack, but faster\.
@@ -19,16 +19,20 @@ Recursively search for PATTERN in PATH\. Like grep or ack, but faster\.
 Output results in a format parseable by AckMate \fIhttps://github\.com/protocool/AckMate\fR\.
 .
 .TP
+\fB\-\-[no]affinity\fR
+Set thread affinity (if platform supports it)\. Default is true\.
+.
+.TP
 \fB\-a \-\-all\-types\fR
 Search all files\. This doesn\'t include hidden files, and doesn\'t respect any ignore files\.
 .
 .TP
 \fB\-A \-\-after [LINES]\fR
-Print lines after match\. Defaults to 2\.
+Print lines after match\. If not provided, LINES defaults to 2\.
 .
 .TP
 \fB\-B \-\-before [LINES]\fR
-Print lines before match\. Defaults to 2\.
+Print lines before match\. If not provided, LINES defaults to 2\.
 .
 .TP
 \fB\-\-[no]break\fR
@@ -44,15 +48,15 @@ Print color codes in results\. Enabled by default\.
 .
 .TP
 \fB\-\-color\-line\-number\fR
-Color codes for line numbers\. Defaults to 1;33\.
+Color codes for line numbers\. Default is 1;33\.
 .
 .TP
 \fB\-\-color\-match\fR
-Color codes for result match numbers\. Defaults to 30;43\.
+Color codes for result match numbers\. Default is 30;43\.
 .
 .TP
 \fB\-\-color\-path\fR
-Color codes for path names\. Defaults to 1;32\.
+Color codes for path names\. Default is 1;32\.
 .
 .TP
 \fB\-\-column\fR
@@ -60,11 +64,11 @@ Print column numbers in results\.
 .
 .TP
 \fB\-C \-\-context [LINES]\fR
-Print lines before and after matches\. Defaults to 2\.
+Print lines before and after matches\. Default is 2\.
 .
 .TP
 \fB\-D \-\-debug\fR
-Output ridiculous amounts of debugging info\. Probably not useful\.
+Output ridiculous amounts of debugging info\. Not useful unless you\'re actually debugging\.
 .
 .TP
 \fB\-\-depth NUM\fR
@@ -75,8 +79,8 @@ Search up to NUM directories deep, \-1 for unlimited\. Default is 25\.
 Print file names\. Enabled by default, except when searching a single file\.
 .
 .TP
-\fB\-f \-\-follow\fR
-Follow symlinks\.
+\fB\-f \-\-[no]follow\fR
+Follow symlinks\. Default is false\.
 .
 .TP
 \fB\-F \-\-fixed\-strings\fR
@@ -131,6 +135,18 @@ See \fBFILE TYPES\fR below\.
 Skip the rest of a file after NUM matches\. Default is 0, which never skips\.
 .
 .TP
+\fB\-\-[no]mmap\fR
+Toggle use of memory\-mapped I/O\. Defaults to true on platforms where \fBmmap()\fR is faster than \fBread()\fR\. (All but macOS\.)
+.
+.TP
+\fB\-\-[no]multiline\fR
+Match regexes across newlines\. Enabled by default\.
+.
+.TP
+\fB\-n \-\-norecurse\fR
+Don\'t recurse into directories\.
+.
+.TP
 \fB\-\-[no]numbers\fR
 Print line numbers\. Default is to omit line numbers when searching streams\.
 .
@@ -139,19 +155,27 @@ Print line numbers\. Default is to omit line numbers when searching streams\.
 Print only the matching part of the lines\.
 .
 .TP
-\fB\-p \-\-path\-to\-agignore STRING\fR
-Provide a path to a specific \.agignore file\.
+\fB\-\-one\-device\fR
+When recursing directories, don\'t scan dirs that reside on other storage devices\. This lets you avoid scanning slow network mounts\. This feature is not supported on all platforms\.
+.
+.TP
+\fB\-p \-\-path\-to\-ignore STRING\fR
+Provide a path to a specific \.ignore file\.
 .
 .TP
 \fB\-\-pager COMMAND\fR
-Use a pager such as less\. Use \fB\-\-nopager\fR to override\. This option is also ignored if output is piped to another program\.
+Use a pager such as \fBless\fR\. Use \fB\-\-nopager\fR to override\. This option is also ignored if output is piped to another program\.
+.
+.TP
+\fB\-\-parallel\fR
+Parse the input stream as a search term, not data to search\. This is meant to be used with tools such as GNU parallel\. For example: \fBecho "foo\enbar\enbaz" | parallel "ag {} \."\fR will run 3 instances of ag, searching the current directory for "foo", "bar", and "baz"\.
 .
 .TP
 \fB\-\-print\-long\-lines\fR
 Print matches on very long lines (> 2k characters by default)\.
 .
 .TP
-\fB\-\-passthrough\fR
+\fB\-\-passthrough \-\-passthru\fR
 When searching a stream, print all lines even if they don\'t match\.
 .
 .TP
@@ -159,6 +183,10 @@ When searching a stream, print all lines even if they don\'t match\.
 Do not parse PATTERN as a regular expression\. Try to match it literally\.
 .
 .TP
+\fB\-r \-\-recurse\fR
+Recurse into directories when searching\. Default is true\.
+.
+.TP
 \fB\-s \-\-case\-sensitive\fR
 Match case\-sensitively\.
 .
@@ -179,60 +207,66 @@ Suppress all log messages, including errors\.
 Print stats (files scanned, time taken, etc)\.
 .
 .TP
+\fB\-\-stats\-only\fR
+Print stats (files scanned, time taken, etc) and nothing else\.
+.
+.TP
 \fB\-t \-\-all\-text\fR
 Search all text files\. This doesn\'t include hidden files\.
 .
 .TP
 \fB\-u \-\-unrestricted\fR
-Search \fIall\fR files\. This ignores \.agignore, \.gitignore, etc\. It searches binary and hidden files as well\.
+Search \fIall\fR files\. This ignores \.ignore, \.gitignore, etc\. It searches binary and hidden files as well\.
 .
 .TP
 \fB\-U \-\-skip\-vcs\-ignores\fR
-Ignore VCS ignore files (\.gitignore, \.hgignore, svn:ignore), but still use \.agignore\.
+Ignore VCS ignore files (\.gitignore, \.hgignore), but still use \.ignore\.
 .
 .TP
 \fB\-v \-\-invert\-match\fR
 Match every line \fInot\fR containing the specified pattern\.
 .
 .TP
+\fB\-V \-\-version\fR
+Print version info\.
+.
+.TP
 \fB\-\-vimgrep\fR
-Output results like vim\'s \fB:vimgrep /pattern/g\fR would (it reports every match on the line)\.
-.
-.br
-Here\'s a ~/\.vimrc configuration example:
+Output results in the same form as Vim\'s \fB:vimgrep /pattern/g\fR
 .
 .IP
-\fBset grepprg=ag\e \-\-vimgrep\e $*\fR
-.
-.br
-\fBset grepformat=%f:%l:%c:%m\fR
+Here is a ~/\.vimrc configuration example:
 .
 .IP
-Then use \fB:grep\fR to grep for something\.
+\fBset grepprg=ag\e \-\-vimgrep\e $*\fR \fBset grepformat=%f:%l:%c:%m\fR
 .
-.br
-Then use \fB:copen\fR, \fB:cn\fR, \fB:cp\fR, etc\. to navigate through the matches\.
+.IP
+Then use \fB:grep\fR to grep for something\. Then use \fB:copen\fR, \fB:cn\fR, \fB:cp\fR, etc\. to navigate through the matches\.
 .
 .TP
 \fB\-w \-\-word\-regexp\fR
 Only match whole words\.
 .
 .TP
+\fB\-\-workers NUM\fR
+Use NUM worker threads\. Default is the number of CPU cores, with a max of 8\.
+.
+.TP
 \fB\-z \-\-search\-zip\fR
-Search contents of compressed files\.
+Search contents of compressed files\. Currently, gz and xz are supported\. This option requires that ag is built with lzma and zlib\.
 .
 .TP
 \fB\-0 \-\-null \-\-print0\fR
 Separate the filenames with \fB\e0\fR, rather than \fB\en\fR: this allows \fBxargs \-0 <command>\fR to correctly process filenames containing spaces or newlines\.
 .
 .SH "FILE TYPES"
-It is possible to restrict the types of files searched\. For example, passing \fB\-\-html\fR as the \fBfile\-types\fR parameter will search only files with the extensions \fBhtm\fR, \fBhtml\fR, \fBshtml\fR or \fBxhtml\fR\. For a list of supported \fBfile\-types\fR run \fBag \-\-list\-file\-types\fR\.
+It is possible to restrict the types of files searched\. For example, passing \fB\-\-html\fR will search only files with the extensions \fBhtm\fR, \fBhtml\fR, \fBshtml\fR or \fBxhtml\fR\. For a list of supported types, run \fBag \-\-list\-file\-types\fR\.
 .
 .SH "IGNORING FILES"
-By default, ag will ignore files whose names match patterns in \.gitignore, \.hgignore, or \.agignore\. These files can be anywhere in the directories being searched\. Ag also ignores files matched by the svn:ignore property if \fBsvn \-\-version\fR is 1\.6 or older\. Finally, ag looks in $HOME/\.agignore for ignore patterns\. Binary files are ignored by default as well\.
+By default, ag will ignore files whose names match patterns in \.gitignore, \.hgignore, or \.ignore\. These files can be anywhere in the directories being searched\. Binary files are ignored by default as well\. Finally, ag looks in $HOME/\.agignore for ignore patterns\.
 .
 .P
-If you want to ignore \.gitignore, \.hgignore, and svn:ignore, but still take \.agignore into account, use \fB\-U\fR\.
+If you want to ignore \.gitignore and \.hgignore, but still take \.ignore into account, use \fB\-U\fR\.
 .
 .P
 Use the \fB\-t\fR option to search all text files; \fB\-a\fR to search all files; and \fB\-u\fR to search all, including hidden files\.
@@ -243,5 +277,11 @@ Use the \fB\-t\fR option to search all text files; \fB\-a\fR to search all files
 .P
 \fBag foo /bar/\fR: Find matches for "foo" in path /bar/\.
 .
+.P
+\fBag \-\- \-\-foo\fR: Find matches for "\-\-foo" in the current directory\. (As with most UNIX command line utilities, "\-\-" is used to signify that the remaining arguments should not be treated as options\.)
+.
+.SH "ABOUT"
+ag was originally created by Geoff Greer\. More information (and the latest release) can be found at http://geoff\.greer\.fm/ag
+.
 .SH "SEE ALSO"
 grep(1)

doc/ag.1.md

@@ -3,7 +3,7 @@ ag(1) -- The Silver Searcher. Like ack, but faster.
 
 ## SYNOPSIS
 
-`ag` [<file-type>] [<options>] PATTERN [PATH]
+`ag` [_options_] _pattern_ [_path ..._]
 
 ## DESCRIPTION
 
@@ -14,14 +14,17 @@ Recursively search for PATTERN in PATH. Like grep or ack, but faster.
   * `--ackmate`:
     Output results in a format parseable by [AckMate](https://github.com/protocool/AckMate).
 
+  * `--[no]affinity`:
+    Set thread affinity (if platform supports it). Default is true.
+
   * `-a --all-types`:
     Search all files. This doesn't include hidden files, and doesn't respect any ignore files.
 
   * `-A --after [LINES]`:
-    Print lines after match. Defaults to 2.
+    Print lines after match. If not provided, LINES defaults to 2.
 
   * `-B --before [LINES]`:
-    Print lines before match. Defaults to 2.
+    Print lines before match. If not provided, LINES defaults to 2.
 
   * `--[no]break`:
     Print a newline between matches in different files. Enabled by default.
@@ -35,22 +38,22 @@ Recursively search for PATTERN in PATH. Like grep or ack, but faster.
     Print color codes in results. Enabled by default.
 
   * `--color-line-number`:
-    Color codes for line numbers. Defaults to 1;33.
+    Color codes for line numbers. Default is 1;33.
 
   * `--color-match`:
-    Color codes for result match numbers. Defaults to 30;43.
+    Color codes for result match numbers. Default is 30;43.
 
   * `--color-path`:
-    Color codes for path names. Defaults to 1;32.
+    Color codes for path names. Default is 1;32.
 
   * `--column`:
     Print column numbers in results.
 
   * `-C --context [LINES]`:
-    Print lines before and after matches. Defaults to 2.
+    Print lines before and after matches. Default is 2.
 
   * `-D --debug`:
-    Output ridiculous amounts of debugging info. Probably not useful.
+    Output ridiculous amounts of debugging info. Not useful unless you're actually debugging.
 
   * `--depth NUM`:
     Search up to NUM directories deep, -1 for unlimited. Default is 25.
@@ -58,8 +61,8 @@ Recursively search for PATTERN in PATH. Like grep or ack, but faster.
   * `--[no]filename`:
     Print file names. Enabled by default, except when searching a single file.
 
-  * `-f --follow`:
-    Follow symlinks.
+  * `-f --[no]follow`:
+    Follow symlinks. Default is false.
 
   * `-F --fixed-strings`:
     Alias for --literal for compatibility with grep.
@@ -105,31 +108,52 @@ Recursively search for PATTERN in PATH. Like grep or ack, but faster.
   * `-m --max-count NUM`:
     Skip the rest of a file after NUM matches. Default is 0, which never skips.
 
+  * `--[no]mmap`:
+    Toggle use of memory-mapped I/O. Defaults to true on platforms where
+    `mmap()` is faster than `read()`. (All but macOS.)
+
   * `--[no]multiline`:
     Match regexes across newlines. Enabled by default.
 
+  * `-n --norecurse`:
+    Don't recurse into directories.
+
   * `--[no]numbers`:
     Print line numbers. Default is to omit line numbers when searching streams.
 
   * `-o --only-matching`:
     Print only the matching part of the lines.
 
-  * `-p --path-to-agignore STRING`:
-    Provide a path to a specific .agignore file.
+  * `--one-device`:
+    When recursing directories, don't scan dirs that reside on other storage
+    devices. This lets you avoid scanning slow network mounts.
+    This feature is not supported on all platforms.
+
+  * `-p --path-to-ignore STRING`:
+    Provide a path to a specific .ignore file.
 
   * `--pager COMMAND`:
-    Use a pager such as less. Use `--nopager` to override. This option
+    Use a pager such as `less`. Use `--nopager` to override. This option
     is also ignored if output is piped to another program.
 
+  * `--parallel`:
+    Parse the input stream as a search term, not data to search. This is meant
+    to be used with tools such as GNU parallel. For example:
+    `echo "foo\nbar\nbaz" | parallel "ag {} ."` will run 3 instances of ag,
+    searching the current directory for "foo", "bar", and "baz".
+
   * `--print-long-lines`:
     Print matches on very long lines (> 2k characters by default).
 
-  * `--passthrough`:
+  * `--passthrough --passthru`:
     When searching a stream, print all lines even if they don't match.
 
   * `-Q --literal`:
     Do not parse PATTERN as a regular expression. Try to match it literally.
 
+  * `-r --recurse`:
+    Recurse into directories when searching. Default is true.
+
   * `-s --case-sensitive`:
     Match case-sensitively.
 
@@ -146,23 +170,30 @@ Recursively search for PATTERN in PATH. Like grep or ack, but faster.
   * `--stats`:
     Print stats (files scanned, time taken, etc).
 
+  * `--stats-only`:
+    Print stats (files scanned, time taken, etc) and nothing else.
+
   * `-t --all-text`:
     Search all text files. This doesn't include hidden files.
 
   * `-u --unrestricted`:
-    Search *all* files. This ignores .agignore, .gitignore, etc. It searches
+    Search *all* files. This ignores .ignore, .gitignore, etc. It searches
     binary and hidden files as well.
 
   * `-U --skip-vcs-ignores`:
-    Ignore VCS ignore files (.gitignore, .hgignore, svn:ignore), but still
-    use .agignore.
+    Ignore VCS ignore files (.gitignore, .hgignore), but still
+    use .ignore.
 
   * `-v --invert-match`:
     Match every line *not* containing the specified pattern.
 
+  * `-V --version`:
+    Print version info.
+
   * `--vimgrep`:
-    Output results like vim's `:vimgrep /pattern/g` would (it reports every match on the line).
-    Here's a ~/.vimrc configuration example:
+    Output results in the same form as Vim's `:vimgrep /pattern/g`
+
+    Here is a ~/.vimrc configuration example:
 
     `set grepprg=ag\ --vimgrep\ $*`
     `set grepformat=%f:%l:%c:%m`
@@ -173,8 +204,15 @@ Recursively search for PATTERN in PATH. Like grep or ack, but faster.
   * `-w --word-regexp`:
     Only match whole words.
 
+  * `--workers NUM`:
+    Use NUM worker threads. Default is the number of CPU cores, with a max of 8.
+
+  * `-W --width NUM`:
+    Truncate match lines after NUM characters.
+
   * `-z --search-zip`:
-    Search contents of compressed files.
+    Search contents of compressed files. Currently, gz and xz are supported.
+    This option requires that ag is built with lzma and zlib.
 
   * `-0 --null --print0`:
     Separate the filenames with `\0`, rather than `\n`:
@@ -185,20 +223,18 @@ Recursively search for PATTERN in PATH. Like grep or ack, but faster.
 ## FILE TYPES
 
 It is possible to restrict the types of files searched. For example, passing
-`--html` as the `file-types` parameter will search only files with the
-extensions `htm`, `html`, `shtml` or `xhtml`. For a list of supported `file-types`
-run `ag --list-file-types`.
+`--html` will search only files with the extensions `htm`, `html`, `shtml`
+or `xhtml`. For a list of supported types, run `ag --list-file-types`.
 
 ## IGNORING FILES
 
 By default, ag will ignore files whose names match patterns in .gitignore,
-.hgignore, or .agignore. These files can be anywhere in the directories being
-searched. Ag also ignores files matched by the svn:ignore property if `svn
---version` is 1.6 or older.  Finally, ag looks in $HOME/.agignore for ignore
-patterns. Binary files are ignored by default as well.
+.hgignore, or .ignore. These files can be anywhere in the directories being
+searched. Binary files are ignored by default as well. Finally, ag looks in
+$HOME/.agignore for ignore patterns.
 
-If you want to ignore .gitignore, .hgignore, and svn:ignore, but still take
-.agignore into account, use `-U`.
+If you want to ignore .gitignore and .hgignore, but still take .ignore into
+account, use `-U`.
 
 Use the `-t` option to search all text files; `-a` to search all files; and `-u`
 to search all, including hidden files.
@@ -211,6 +247,16 @@ to search all, including hidden files.
 `ag foo /bar/`:
   Find matches for "foo" in path /bar/.
 
+`ag -- --foo`:
+  Find matches for "--foo" in the current directory. (As with most UNIX command
+  line utilities, "--" is used to signify that the remaining arguments should
+  not be treated as options.)
+
+## ABOUT
+
+ag was originally created by Geoff Greer. More information (and the latest
+release) can be found at http://geoff.greer.fm/ag
+
 ## SEE ALSO
 
 grep(1)

format.sh

@@ -22,18 +22,21 @@ then
     exit 1
 fi
 
-SOURCE_FILES=`git ls-files src/`
+SOURCE_FILES=$(git ls-files src/)
 
 if [ "$1" == "reformat" ]
 then
     echo "Reformatting source files"
+    # shellcheck disable=2086
     echo $CLANG_FORMAT -style=file -i $SOURCE_FILES
+    # shellcheck disable=2086
     $CLANG_FORMAT -style=file -i $SOURCE_FILES
     exit 0
 elif  [ "$1" == "test" ]
 then
-    RESULT=`$CLANG_FORMAT -style=file -output-replacements-xml $SOURCE_FILES | grep -c '<replacement '`
-    if [ $RESULT -eq 0 ]
+    # shellcheck disable=2086
+    RESULT=$($CLANG_FORMAT -style=file -output-replacements-xml $SOURCE_FILES | grep -c '<replacement ')
+    if [ "$RESULT" -eq 0 ]
     then
         echo "code is formatted correctly :)"
         exit 0

pgo.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -e
+cd "$(dirname "$0")"
+
+make clean
+./build.sh CFLAGS="$CFLAGS -fprofile-generate"
+./ag example ..
+make clean
+./build.sh CFLAGS="$CFLAGS -fprofile-correction -fprofile-use"

sanitize.sh

@@ -0,0 +1,196 @@
+#!/bin/bash
+# Copyright 2016 Allen Wild
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+AVAILABLE_SANITIZERS=(
+    address
+    thread
+    undefined
+    valgrind
+)
+
+DEFAULT_SANITIZERS=(
+    address
+    thread
+    undefined
+)
+
+usage() {
+    cat <<EOF
+Usage: $0 [-h] [valgrind | [SANITIZERS ...]]
+
+    This script recompiles ag using -fsanitize=<SANITIZER> and then runs the test suite.
+    Memory leaks or other errors will be printed in ag's output, thus failing the test.
+
+    Available LLVM sanitizers are: ${AVAILABLE_SANITIZERS[*]}
+
+    The compile-time sanitizers are supported in clang/llvm >= 3.1 and gcc >= 4.8
+    for x86_64 Linux only. clang is preferred and will be used, if available.
+
+    For function names and line numbers in error output traces, llvm-symbolizer needs
+    to be available in PATH or set through ASAN_SYMBOLIZER_PATH.
+
+    If 'valgrind' is passed as the sanitizer, then ag will be run through valgrind
+    without recompiling. If $(dirname $0)/ag doesn't exist, then it will be built.
+
+    WARNING: This script will run "make distclean" and "./configure" to recompile ag
+             once per sanitizer (except for valgrind). If you need to pass additional
+             options to ./configure, put them in the CONFIGOPTS environment variable.
+EOF
+}
+
+vrun() {
+    echo "Running: $*"
+    "$@"
+}
+
+die() {
+    echo "Fatal: $*"
+    exit 1
+}
+
+valid_sanitizer() {
+    for san in "${AVAILABLE_SANITIZERS[@]}"; do
+        if [[ "$1" == "$san" ]]; then
+            return 0
+        fi
+    done
+    return 1
+}
+
+run_sanitizer() {
+    sanitizer=$1
+    if [[ "$sanitizer" == "valgrind" ]]; then
+        run_valgrind
+        return $?
+    fi
+
+    echo -e "\nCompiling for sanitizer '$sanitizer'"
+    [[ -f Makefile ]] && vrun make distclean
+    vrun ./configure $CONFIGOPTS CC=$SANITIZE_CC \
+                     CFLAGS="-g -O0 -fsanitize=$sanitizer $EXTRA_CFLAGS"
+    if [[ $? != 0 ]]; then
+        echo "ERROR: Failed to configure. Try setting CONFIGOPTS?"
+        return 1
+    fi
+
+    vrun make
+    if [[ $? != 0 ]]; then
+        echo "ERROR: failed to build"
+        return 1
+    fi
+
+    echo "Testing with sanitizer '$sanitizer'"
+    vrun make test
+    if [[ $? != 0 ]]; then
+        echo "Tests for sanitizer '$sanitizer' FAIL!"
+        echo "Check the above output for failure information"
+        return 2
+    else
+        echo "Tests for sanitizer '$sanitizer' PASS!"
+        return 0
+    fi
+}
+
+run_valgrind() {
+    echo "Compiling ag normally for use with valgrind"
+    [[ -f Makefile ]] && vrun make distclean
+    vrun ./configure $CONFIGOPTS
+    if [[ $? != 0 ]]; then
+        echo "ERROR: Failed to configure. Try setting CONFIGOPTS?"
+        return 1
+    fi
+
+    vrun make
+    if [[ $? != 0 ]]; then
+        echo "ERROR: failed to build"
+        return 1
+    fi
+
+    echo "Running: AGPROG=\"valgrind -q $PWD/ag\" make test"
+    AGPROG="valgrind -q $PWD/ag" make test
+    if [[ $? != 0 ]]; then
+        echo "Valgrind tests FAIL!"
+        return 1
+    else
+        echo "Valgrind tests PASS!"
+        return 0
+    fi
+}
+
+#### MAIN ####
+run_sanitizers=()
+for opt in "$@"; do
+    if [[ "$opt" == -* ]]; then
+        case opt in
+            -h|--help)
+                usage
+                exit 0
+                ;;
+            *)
+                echo "Unknown option: '$opt'"
+                usage
+                exit 1
+                ;;
+        esac
+    else
+        if valid_sanitizer "$opt"; then
+            run_sanitizers+=("$opt")
+        else
+            echo "Invalid Sanitizer: '$opt'"
+            usage
+            exit 1
+        fi
+    fi
+done
+
+if [[ ${#run_sanitizers[@]} == 0 ]]; then
+    run_sanitizers=(${DEFAULT_SANITIZERS[@]})
+fi
+
+if [[ -n $CC ]]; then
+    echo "Using CC=$CC"
+    SANITIZE_CC="$CC"
+elif which clang &>/dev/null; then
+    SANITIZE_CC="clang"
+else
+    echo "Warning: CC unset and clang not found"
+fi
+
+if [[ -n $CFLAGS ]]; then
+    EXTRA_CFLAGS="$CFLAGS"
+    unset CFLAGS
+fi
+
+if [[ ! -e ./configure ]]; then
+    echo "Warning: ./configure not found. Running autogen"
+    vrun ./autogen.sh || die "autogen.sh failed"
+fi
+
+echo "Running sanitizers: ${run_sanitizers[*]}"
+failedsan=()
+for san in "${run_sanitizers[@]}"; do
+    run_sanitizer $san
+    if [[ $? != 0 ]]; then
+        failedsan+=($san)
+    fi
+done
+
+if [[ ${#failedsan[@]} == 0 ]]; then
+    echo "All sanitizers PASSED"
+    exit 0
+else
+    echo "The following sanitizers FAILED: ${failedsan[*]}"
+    exit ${#failedsan[@]}
+fi

src/decompress.h

@@ -1,6 +1,8 @@
 #ifndef DECOMPRESS_H
 #define DECOMPRESS_H
 
+#include <stdio.h>
+
 #include "config.h"
 #include "log.h"
 #include "options.h"
@@ -16,4 +18,9 @@ typedef enum {
 ag_compression_type is_zipped(const void *buf, const int buf_len);
 
 void *decompress(const ag_compression_type zip_type, const void *buf, const int buf_len, const char *dir_full_path, int *new_buf_len);
+
+#if HAVE_FOPENCOOKIE
+FILE *decompress_open(int fd, const char *mode, ag_compression_type ctype);
+#endif
+
 #endif

src/ignore.c

@@ -1,5 +1,6 @@
 #include <ctype.h>
 #include <dirent.h>
+#include <limits.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -19,6 +20,8 @@
 const int fnmatch_flags = FNM_PATHNAME;
 #endif
 
+ignores *root_ignores;
+
 /* TODO: build a huge-ass list of files we want to ignore by default (build cache stuff, pyc files, etc) */
 
 const char *evil_hardcoded_ignore_files[] = {
@@ -27,13 +30,12 @@ const char *evil_hardcoded_ignore_files[] = {
     NULL
 };
 
-/* Warning: changing the first string will break skip_vcs_ignores. */
+/* Warning: changing the first two strings will break skip_vcs_ignores. */
 const char *ignore_pattern_files[] = {
-    ".agignore",
+    ".ignore",
     ".gitignore",
     ".git/info/exclude",
     ".hgignore",
-    ".svn",
     NULL
 };
 
@@ -51,6 +53,8 @@ ignores *init_ignore(ignores *parent, const char *dirname, const size_t dirname_
     ig->slash_names_len = 0;
     ig->regexes = NULL;
     ig->regexes_len = 0;
+    ig->invert_regexes = NULL;
+    ig->invert_regexes_len = 0;
     ig->slash_regexes = NULL;
     ig->slash_regexes_len = 0;
     ig->dirname = dirname;
@@ -84,6 +88,7 @@ void cleanup_ignore(ignores *ig) {
     free_strings(ig->names, ig->names_len);
     free_strings(ig->slash_names, ig->slash_names_len);
     free_strings(ig->regexes, ig->regexes_len);
+    free_strings(ig->invert_regexes, ig->invert_regexes_len);
     free_strings(ig->slash_regexes, ig->slash_regexes_len);
     if (ig->abs_path) {
         free(ig->abs_path);
@@ -115,15 +120,21 @@ void add_ignore_pattern(ignores *ig, const char *pattern) {
     char ***patterns_p;
     size_t *patterns_len;
     if (is_fnmatch(pattern)) {
-        if (pattern[0] == '*' && pattern[1] == '.' && !(is_fnmatch(pattern + 2))) {
+        if (pattern[0] == '*' && pattern[1] == '.' && strchr(pattern + 2, '.') && !is_fnmatch(pattern + 2)) {
             patterns_p = &(ig->extensions);
             patterns_len = &(ig->extensions_len);
             pattern += 2;
+            pattern_len -= 2;
         } else if (pattern[0] == '/') {
             patterns_p = &(ig->slash_regexes);
             patterns_len = &(ig->slash_regexes_len);
             pattern++;
             pattern_len--;
+        } else if (pattern[0] == '!') {
+            patterns_p = &(ig->invert_regexes);
+            patterns_len = &(ig->invert_regexes_len);
+            pattern++;
+            pattern_len--;
         } else {
             patterns_p = &(ig->regexes);
             patterns_len = &(ig->regexes_len);
@@ -186,76 +197,6 @@ void load_ignore_patterns(ignores *ig, const char *path) {
     fclose(fp);
 }
 
-void load_svn_ignore_patterns(ignores *ig, const char *path) {
-    FILE *fp = NULL;
-    char *dir_prop_base;
-    ag_asprintf(&dir_prop_base, "%s/%s", path, SVN_DIR_PROP_BASE);
-
-    fp = fopen(dir_prop_base, "r");
-    if (fp == NULL) {
-        log_debug("Skipping svn ignore file %s", dir_prop_base);
-        free(dir_prop_base);
-        return;
-    }
-
-    char *entry = NULL;
-    size_t entry_len = 0;
-    char *key = ag_malloc(32); /* Sane start for max key length. */
-    size_t key_len = 0;
-    size_t bytes_read = 0;
-    char *entry_line;
-    size_t line_len;
-    int matches;
-
-    while (fscanf(fp, "K %zu\n", &key_len) == 1) {
-        key = ag_realloc(key, key_len + 1);
-        bytes_read = fread(key, 1, key_len, fp);
-        key[key_len] = '\0';
-        matches = fscanf(fp, "\nV %zu\n", &entry_len);
-        if (matches != 1) {
-            log_debug("Unable to parse svnignore file %s: fscanf() got %i matches, expected 1.", dir_prop_base, matches);
-            goto cleanup;
-        }
-
-        if (strncmp(SVN_PROP_IGNORE, key, bytes_read) != 0) {
-            log_debug("key is %s, not %s. skipping %u bytes", key, SVN_PROP_IGNORE, entry_len);
-            /* Not the key we care about. fseek and repeat */
-            fseek(fp, entry_len + 1, SEEK_CUR); /* +1 to account for newline. yes I know this is hacky */
-            continue;
-        }
-        /* Aww yeah. Time to ignore stuff */
-        entry = ag_malloc(entry_len + 1);
-        bytes_read = fread(entry, 1, entry_len, fp);
-        entry[bytes_read] = '\0';
-        log_debug("entry: %s", entry);
-        break;
-    }
-    if (entry == NULL) {
-        goto cleanup;
-    }
-    char *patterns = entry;
-    size_t patterns_len = strlen(patterns);
-    while (*patterns != '\0' && patterns < (entry + bytes_read)) {
-        for (line_len = 0; line_len < patterns_len; line_len++) {
-            if (patterns[line_len] == '\n') {
-                break;
-            }
-        }
-        if (line_len > 0) {
-            entry_line = ag_strndup(patterns, line_len);
-            add_ignore_pattern(ig, entry_line);
-            free(entry_line);
-        }
-        patterns += line_len + 1;
-        patterns_len -= line_len + 1;
-    }
-    free(entry);
-cleanup:
-    free(dir_prop_base);
-    free(key);
-    fclose(fp);
-}
-
 static int ackmate_dir_match(const char *dir_name) {
     if (opts.ackmate_dir_filter == NULL) {
         return 0;
@@ -267,6 +208,7 @@ static int ackmate_dir_match(const char *dir_name) {
 /* This is the hottest code in Ag. 10-15% of all execution time is spent here */
 static int path_ignore_search(const ignores *ig, const char *path, const char *filename) {
     char *temp;
+    int temp_start_pos;
     size_t i;
     int match_pos;
 
@@ -277,9 +219,12 @@ static int path_ignore_search(const ignores *ig, const char *path, const char *f
     }
 
     ag_asprintf(&temp, "%s/%s", path[0] == '.' ? path + 1 : path, filename);
+    //ig->abs_path has its leading slash stripped, so we have to strip the leading slash
+    //of temp as well
+    temp_start_pos = (temp[0] == '/') ? 1 : 0;
 
-    if (strncmp(temp, ig->abs_path, ig->abs_path_len) == 0) {
-        char *slash_filename = temp + ig->abs_path_len;
+    if (strncmp(temp + temp_start_pos, ig->abs_path, ig->abs_path_len) == 0) {
+        char *slash_filename = temp + temp_start_pos + ig->abs_path_len;
         if (slash_filename[0] == '/') {
             slash_filename++;
         }
@@ -320,6 +265,15 @@ static int path_ignore_search(const ignores *ig, const char *path, const char *f
         }
     }
 
+    for (i = 0; i < ig->invert_regexes_len; i++) {
+        if (fnmatch(ig->invert_regexes[i], filename, fnmatch_flags) == 0) {
+            log_debug("file %s not ignored because name matches regex pattern !%s", filename, ig->invert_regexes[i]);
+            free(temp);
+            return 0;
+        }
+        log_debug("pattern !%s doesn't match file %s", ig->invert_regexes[i], filename);
+    }
+
     for (i = 0; i < ig->regexes_len; i++) {
         if (fnmatch(ig->regexes[i], filename, fnmatch_flags) == 0) {
             log_debug("file %s ignored because name matches regex pattern %s", filename, ig->regexes[i]);
@@ -337,24 +291,11 @@ static int path_ignore_search(const ignores *ig, const char *path, const char *f
 /* This function is REALLY HOT. It gets called for every file */
 int filename_filter(const char *path, const struct dirent *dir, void *baton) {
     const char *filename = dir->d_name;
-/* TODO: don't call strlen on filename every time we call filename_filter() */
-#ifdef HAVE_DIRENT_DNAMLEN
-    size_t filename_len = dir->d_namlen;
-#else
-    size_t filename_len = strlen(filename);
-#endif
-    size_t i;
-    scandir_baton_t *scandir_baton = (scandir_baton_t *)baton;
-    const ignores *ig = scandir_baton->ig;
-    const char *base_path = scandir_baton->base_path;
-    const size_t base_path_len = scandir_baton->base_path_len;
-    const char *path_start = path;
-    char *temp;
-
     if (!opts.search_hidden_files && filename[0] == '.') {
         return 0;
     }
 
+    size_t i;
     for (i = 0; evil_hardcoded_ignore_files[i] != NULL; i++) {
         if (strcmp(filename, evil_hardcoded_ignore_files[i]) == 0) {
             return 0;
@@ -367,19 +308,16 @@ int filename_filter(const char *path, const struct dirent *dir, void *baton) {
     }
 
     if (is_named_pipe(path, dir)) {
-        log_debug("%s ignored because it's a named pipe", path);
+        log_debug("%s ignored because it's a named pipe or socket", path);
         return 0;
     }
 
-    if (opts.search_all_files && !opts.path_to_agignore) {
+    if (opts.search_all_files && !opts.path_to_ignore) {
         return 1;
     }
 
-    for (i = 0; base_path[i] == path[i] && i < base_path_len; i++) {
-        /* base_path always ends with "/\0" while path doesn't, so this is safe */
-        path_start = path + i + 2;
-    }
-    log_debug("path_start %s filename %s", path_start, filename);
+    scandir_baton_t *scandir_baton = (scandir_baton_t *)baton;
+    const char *path_start = scandir_baton->path_start;
 
     const char *extension = strchr(filename, '.');
     if (extension) {
@@ -392,12 +330,23 @@ int filename_filter(const char *path, const struct dirent *dir, void *baton) {
         }
     }
 
-    while (ig != NULL) {
-        if (strncmp(filename, "./", 2) == 0) {
-            filename++;
-            filename_len--;
-        }
+#ifdef HAVE_DIRENT_DNAMLEN
+    size_t filename_len = dir->d_namlen;
+#else
+    size_t filename_len = 0;
+#endif
 
+    if (strncmp(filename, "./", 2) == 0) {
+#ifndef HAVE_DIRENT_DNAMLEN
+        filename_len = strlen(filename);
+#endif
+        filename++;
+        filename_len--;
+    }
+
+    const ignores *ig = scandir_baton->ig;
+
+    while (ig != NULL) {
         if (extension) {
             int match_pos = binary_search(extension, ig->extensions, 0, ig->extensions_len);
             if (match_pos >= 0) {
@@ -410,12 +359,20 @@ int filename_filter(const char *path, const struct dirent *dir, void *baton) {
             return 0;
         }
 
-        if (is_directory(path, dir) && filename[filename_len - 1] != '/') {
-            ag_asprintf(&temp, "%s/", filename);
-            int rv = path_ignore_search(ig, path_start, temp);
-            free(temp);
-            if (rv) {
-                return 0;
+        if (is_directory(path, dir)) {
+#ifndef HAVE_DIRENT_DNAMLEN
+            if (!filename_len) {
+                filename_len = strlen(filename);
+            }
+#endif
+            if (filename[filename_len - 1] != '/') {
+                char *temp;
+                ag_asprintf(&temp, "%s/", filename);
+                int rv = path_ignore_search(ig, path_start, temp);
+                free(temp);
+                if (rv) {
+                    return 0;
+                }
             }
         }
         ig = ig->parent;

src/ignore.h

@@ -4,10 +4,6 @@
 #include <dirent.h>
 #include <sys/types.h>
 
-#define SVN_DIR_PROP_BASE "dir-prop-base"
-#define SVN_DIR ".svn"
-#define SVN_PROP_IGNORE "svn:ignore"
-
 struct ignores {
     char **extensions; /* File extensions to ignore */
     size_t extensions_len;
@@ -19,6 +15,8 @@ struct ignores {
 
     char **regexes; /* For patterns that need fnmatch */
     size_t regexes_len;
+    char **invert_regexes; /* For "!" patterns */
+    size_t invert_regexes_len;
     char **slash_regexes;
     size_t slash_regexes_len;
 
@@ -31,7 +29,7 @@ struct ignores {
 };
 typedef struct ignores ignores;
 
-ignores *root_ignores;
+extern ignores *root_ignores;
 
 extern const char *evil_hardcoded_ignore_files[];
 extern const char *ignore_pattern_files[];
@@ -42,7 +40,6 @@ void cleanup_ignore(ignores *ig);
 void add_ignore_pattern(ignores *ig, const char *pattern);
 
 void load_ignore_patterns(ignores *ig, const char *path);
-void load_svn_ignore_patterns(ignores *ig, const char *path);
 
 int filename_filter(const char *path, const struct dirent *dir, void *baton);
 

src/lang.c

@@ -7,38 +7,67 @@
 lang_spec_t langs[] = {
     { "actionscript", { "as", "mxml" } },
     { "ada", { "ada", "adb", "ads" } },
+    { "asciidoc", { "adoc", "ad", "asc", "asciidoc" } },
+    { "apl", { "apl" } },
     { "asm", { "asm", "s" } },
+    { "asp", { "asp", "asa", "aspx", "asax", "ashx", "ascx", "asmx" } },
+    { "aspx", { "asp", "asa", "aspx", "asax", "ashx", "ascx", "asmx" } },
     { "batch", { "bat", "cmd" } },
+    { "bazel", { "bazel" } },
+    { "bitbake", { "bb", "bbappend", "bbclass", "inc" } },
     { "cc", { "c", "h", "xs" } },
     { "cfmx", { "cfc", "cfm", "cfml" } },
-    { "clojure", { "clj", "cljs", "cljc", "cljx" } },
+    { "chpl", { "chpl" } },
+    { "clojure", { "clj", "cljs", "cljc", "cljx", "edn" } },
     { "coffee", { "coffee", "cjsx" } },
-    { "cpp", { "cpp", "cc", "C", "cxx", "m", "hpp", "hh", "h", "H", "hxx" } },
+    { "config", { "config" } },
+    { "coq", { "coq", "g", "v" } },
+    { "cpp", { "cpp", "cc", "C", "cxx", "m", "hpp", "hh", "h", "H", "hxx", "tpp" } },
+    { "crystal", { "cr", "ecr" } },
     { "csharp", { "cs" } },
+    { "cshtml", { "cshtml" } },
     { "css", { "css" } },
-    { "delphi", { "pas", "int", "dfm", "nfm", "dof", "dpk", "dproj", "groupproj", "bdsgroup", "bdsproj" } },
+    { "cython", { "pyx", "pxd", "pxi" } },
+    { "delphi", { "pas", "int", "dfm", "nfm", "dof", "dpk", "dpr", "dproj", "groupproj", "bdsgroup", "bdsproj" } },
+    { "dlang", { "d", "di" } },
+    { "dot", { "dot", "gv" } },
+    { "dts", { "dts", "dtsi" } },
     { "ebuild", { "ebuild", "eclass" } },
     { "elisp", { "el" } },
-    { "elixir", { "ex", "exs" } },
+    { "elixir", { "ex", "eex", "exs" } },
+    { "elm", { "elm" } },
     { "erlang", { "erl", "hrl" } },
-    { "fortran", { "f", "f77", "f90", "f95", "f03", "for", "ftn", "fpp" } },
+    { "factor", { "factor" } },
+    { "fortran", { "f", "F", "f77", "f90", "F90", "f95", "f03", "for", "ftn", "fpp", "FPP" } },
     { "fsharp", { "fs", "fsi", "fsx" } },
     { "gettext", { "po", "pot", "mo" } },
+    { "glsl", { "vert", "tesc", "tese", "geom", "frag", "comp" } },
     { "go", { "go" } },
-    { "groovy", { "groovy", "gtmpl", "gpp", "grunit" } },
+    { "gradle", { "gradle" } },
+    { "groovy", { "groovy", "gtmpl", "gpp", "grunit", "gradle" } },
     { "haml", { "haml" } },
-    { "haskell", { "hs", "lhs" } },
+    { "handlebars", { "hbs" } },
+    { "haskell", { "hs", "hsig", "lhs" } },
+    { "haxe", { "hx" } },
     { "hh", { "h" } },
     { "html", { "htm", "html", "shtml", "xhtml" } },
+    { "idris", { "idr", "ipkg", "lidr" } },
     { "ini", { "ini" } },
+    { "ipython", { "ipynb" } },
+    { "isabelle", { "thy" } },
+    { "j", { "ijs" } },
     { "jade", { "jade" } },
     { "java", { "java", "properties" } },
-    { "js", { "js", "jsx" } },
+    { "jinja2", { "j2" } },
+    { "js", { "es6", "js", "jsx", "vue" } },
     { "json", { "json" } },
-    { "jsp", { "jsp", "jspx", "jhtm", "jhtml" } },
+    { "jsp", { "jsp", "jspx", "jhtm", "jhtml", "jspf", "tag", "tagf" } },
+    { "julia", { "jl" } },
+    { "kotlin", { "kt" } },
     { "less", { "less" } },
     { "liquid", { "liquid" } },
     { "lisp", { "lisp", "lsp" } },
+    { "log", { "log" } },
     { "lua", { "lua" } },
     { "m4", { "m4" } },
     { "make", { "Makefiles", "mk", "mak" } },
@@ -47,48 +76,73 @@ lang_spec_t langs[] = {
     { "mason", { "mas", "mhtml", "mpl", "mtxt" } },
     { "matlab", { "m" } },
     { "mathematica", { "m", "wl" } },
+    { "md", { "markdown", "mdown", "mdwn", "mkdn", "mkd", "md" } },
     { "mercury", { "m", "moo" } },
+    { "naccess", { "asa", "rsa" } },
     { "nim", { "nim" } },
+    { "nix", { "nix" } },
     { "objc", { "m", "h" } },
     { "objcpp", { "mm", "h" } },
     { "ocaml", { "ml", "mli", "mll", "mly" } },
     { "octave", { "m" } },
+    { "org", { "org" } },
     { "parrot", { "pir", "pasm", "pmc", "ops", "pod", "pg", "tg" } },
+    { "pdb", { "pdb" } },
     { "perl", { "pl", "pm", "pm6", "pod", "t" } },
     { "php", { "php", "phpt", "php3", "php4", "php5", "phtml" } },
     { "pike", { "pike", "pmod" } },
-    { "plone", { "pt", "cpt", "metadata", "cpy", "py" } },
+    { "plist", { "plist" } },
+    { "plone", { "pt", "cpt", "metadata", "cpy", "py", "xml", "zcml" } },
+    { "powershell", { "ps1" } },
+    { "proto", { "proto" } },
+    { "ps1", { "ps1" } },
+    { "pug", { "pug" } },
     { "puppet", { "pp" } },
     { "python", { "py" } },
+    { "qml", { "qml" } },
     { "racket", { "rkt", "ss", "scm" } },
-    { "rake", { "Rakefiles" } },
+    { "rake", { "Rakefile" } },
+    { "razor", { "cshtml" } },
+    { "restructuredtext", { "rst" } },
     { "rs", { "rs" } },
-    { "r", { "R", "Rmd", "Rnw", "Rtex", "Rrst" } },
+    { "r", { "r", "R", "Rmd", "Rnw", "Rtex", "Rrst" } },
+    { "rdoc", { "rdoc" } },
     { "ruby", { "rb", "rhtml", "rjs", "rxml", "erb", "rake", "spec" } },
     { "rust", { "rs" } },
     { "salt", { "sls" } },
     { "sass", { "sass", "scss" } },
     { "scala", { "scala" } },
     { "scheme", { "scm", "ss" } },
-    { "shell", { "sh", "bash", "csh", "tcsh", "ksh", "zsh" } },
+    { "shell", { "sh", "bash", "csh", "tcsh", "ksh", "zsh", "fish" } },
     { "smalltalk", { "st" } },
     { "sml", { "sml", "fun", "mlb", "sig" } },
     { "sql", { "sql", "ctl" } },
+    { "stata", { "do", "ado" } },
     { "stylus", { "styl" } },
     { "swift", { "swift" } },
     { "tcl", { "tcl", "itcl", "itk" } },
+    { "terraform", { "tf", "tfvars" } },
     { "tex", { "tex", "cls", "sty" } },
+    { "thrift", { "thrift" } },
+    { "tla", { "tla" } },
     { "tt", { "tt", "tt2", "ttml" } },
+    { "toml", { "toml" } },
+    { "ts", { "ts", "tsx" } },
+    { "twig", { "twig" } },
     { "vala", { "vala", "vapi" } },
     { "vb", { "bas", "cls", "frm", "ctl", "vb", "resx" } },
-    { "velocity", { "vm" } },
-    { "verilog", { "v", "vh", "sv" } },
+    { "velocity", { "vm", "vtl", "vsl" } },
+    { "verilog", { "v", "vh", "sv", "svh" } },
     { "vhdl", { "vhd", "vhdl" } },
     { "vim", { "vim" } },
+    { "vue", { "vue" } },
+    { "wix", { "wxi", "wxs" } },
     { "wsdl", { "wsdl" } },
     { "wadl", { "wadl" } },
-    { "xml", { "xml", "dtd", "xsl", "xslt", "ent" } },
-    { "yaml", { "yaml", "yml" } }
+    { "xml", { "xml", "dtd", "xsl", "xslt", "xsd", "ent", "tld", "plist", "wsdl" } },
+    { "yaml", { "yaml", "yml" } },
+    { "zeek", { "zeek", "bro", "bif" } },
+    { "zephir", { "zep" } }
 };
 
 size_t get_lang_count() {

src/log.c

@@ -1,9 +1,10 @@
-#include <stdio.h>
 #include <stdarg.h>
+#include <stdio.h>
 
 #include "log.h"
 #include "util.h"
 
+pthread_mutex_t print_mtx = PTHREAD_MUTEX_INITIALIZER;
 static enum log_level log_threshold = LOG_LEVEL_ERR;
 
 void set_log_level(enum log_level threshold) {
@@ -43,6 +44,7 @@ void vplog(const unsigned int level, const char *fmt, va_list args) {
         return;
     }
 
+    pthread_mutex_lock(&print_mtx);
     FILE *stream = out_fd;
 
     switch (level) {
@@ -63,6 +65,7 @@ void vplog(const unsigned int level, const char *fmt, va_list args) {
 
     vfprintf(stream, fmt, args);
     fprintf(stream, "\n");
+    pthread_mutex_unlock(&print_mtx);
 }
 
 void plog(const unsigned int level, const char *fmt, ...) {

src/log.h

@@ -3,6 +3,14 @@
 
 #include <stdarg.h>
 
+#include "config.h"
+
+#ifdef HAVE_PTHREAD_H
+#include <pthread.h>
+#endif
+
+extern pthread_mutex_t print_mtx;
+
 enum log_level {
     LOG_LEVEL_DEBUG = 10,
     LOG_LEVEL_MSG = 20,

src/main.c

@@ -1,8 +1,8 @@
+#include <ctype.h>
 #include <pcre.h>
 #include <stdarg.h>
 #include <stdio.h>
 #include <string.h>
-#include <ctype.h>
 #include <sys/time.h>
 #include <unistd.h>
 #ifdef _WIN32
@@ -11,10 +11,18 @@
 
 #include "config.h"
 
+#ifdef HAVE_SYS_CPUSET_H
+#include <sys/cpuset.h>
+#endif
+
 #ifdef HAVE_PTHREAD_H
 #include <pthread.h>
 #endif
 
+#if defined(HAVE_PTHREAD_SETAFFINITY_NP) && defined(__FreeBSD__)
+#include <pthread_np.h>
+#endif
+
 #include "log.h"
 #include "options.h"
 #include "search.h"
@@ -35,6 +43,12 @@ int main(int argc, char **argv) {
     int workers_len;
     int num_cores;
 
+#ifdef HAVE_PLEDGE
+    if (pledge("stdio rpath proc exec", NULL) == -1) {
+        die("pledge: %s", strerror(errno));
+    }
+#endif
+
     set_log_level(LOG_LEVEL_WARN);
 
     work_queue = NULL;
@@ -67,7 +81,7 @@ int main(int argc, char **argv) {
     num_cores = (int)sysconf(_SC_NPROCESSORS_ONLN);
 #endif
 
-    workers_len = num_cores;
+    workers_len = num_cores < 8 ? num_cores : 8;
     if (opts.literal) {
         workers_len--;
     }
@@ -109,6 +123,7 @@ int main(int argc, char **argv) {
         generate_alpha_skip(opts.query, opts.query_len, alpha_skip_lookup, opts.casing == CASE_SENSITIVE);
         find_skip_lookup = NULL;
         generate_find_skip(opts.query, opts.query_len, &find_skip_lookup, opts.casing == CASE_SENSITIVE);
+        generate_hash(opts.query, opts.query_len, h_table, opts.casing == CASE_SENSITIVE);
         if (opts.word_regexp) {
             init_wordchar_table();
             opts.literal_starts_wordchar = is_wordchar(opts.query[0]);
@@ -120,7 +135,7 @@ int main(int argc, char **argv) {
         }
         if (opts.word_regexp) {
             char *word_regexp_query;
-            ag_asprintf(&word_regexp_query, "\\b%s\\b", opts.query);
+            ag_asprintf(&word_regexp_query, "\\b(?:%s)\\b", opts.query);
             free(opts.query);
             opts.query = word_regexp_query;
             opts.query_len = strlen(opts.query);
@@ -137,9 +152,13 @@ int main(int argc, char **argv) {
             if (rv != 0) {
                 die("Error in pthread_create(): %s", strerror(rv));
             }
-#if defined(HAVE_PTHREAD_SETAFFINITY_NP) && defined(USE_CPU_SET)
+#if defined(HAVE_PTHREAD_SETAFFINITY_NP) && (defined(USE_CPU_SET) || defined(HAVE_SYS_CPUSET_H))
             if (opts.use_thread_affinity) {
+#if defined(__linux__) || defined(__midipix__)
                 cpu_set_t cpu_set;
+#elif __FreeBSD__
+                cpuset_t cpu_set;
+#endif
                 CPU_ZERO(&cpu_set);
                 CPU_SET(i % num_cores, &cpu_set);
                 rv = pthread_setaffinity_np(workers[i].thread, sizeof(cpu_set), &cpu_set);
@@ -156,11 +175,17 @@ int main(int argc, char **argv) {
             log_debug("No CPU affinity support.");
 #endif
         }
+
+#ifdef HAVE_PLEDGE
+        if (pledge("stdio rpath", NULL) == -1) {
+            die("pledge: %s", strerror(errno));
+        }
+#endif
         for (i = 0; paths[i] != NULL; i++) {
             log_debug("searching path %s for %s", paths[i], opts.query);
             symhash = NULL;
             ignores *ig = init_ignore(root_ignores, "", 0);
-            struct stat s = {.st_dev = 0 };
+            struct stat s = { .st_dev = 0 };
 #ifndef _WIN32
             /* The device is ignored if opts.one_dev is false, so it's fine
              * to leave it at the default 0
@@ -188,7 +213,7 @@ int main(int argc, char **argv) {
         double time_diff = ((long)stats.time_end.tv_sec * 1000000 + stats.time_end.tv_usec) -
                            ((long)stats.time_start.tv_sec * 1000000 + stats.time_start.tv_usec);
         time_diff /= 1000000;
-        printf("%ld matches\n%ld files contained matches\n%ld files searched\n%ld bytes searched\n%f seconds\n",
+        printf("%zu matches\n%zu files contained matches\n%zu files searched\n%zu bytes searched\n%f seconds\n",
                stats.total_matches, stats.total_file_matches, stats.total_files, stats.total_bytes, time_diff);
         pthread_mutex_destroy(&stats_mtx);
     }

src/options.c

@@ -10,16 +10,18 @@
 
 #include "config.h"
 #include "ignore.h"
-#include "options.h"
 #include "lang.h"
 #include "log.h"
-#include "util.h"
+#include "options.h"
 #include "print.h"
+#include "util.h"
 
 const char *color_line_number = "\033[1;33m"; /* bold yellow */
 const char *color_match = "\033[30;43m";      /* black with yellow background */
 const char *color_path = "\033[1;32m";        /* bold green */
 
+cli_options opts;
+
 /* TODO: try to obey out_fd? */
 void usage(void) {
     printf("\n");
@@ -57,11 +59,14 @@ Output Options:\n\
                           (Enabled by default)\n\
   -C --context [LINES]    Print lines before and after matches (Default: 2)\n\
      --[no]group          Same as --[no]break --[no]heading\n\
-  -g PATTERN              Print filenames matching PATTERN\n\
+  -g --filename-pattern PATTERN\n\
+                          Print filenames matching PATTERN\n\
   -l --files-with-matches Only print filenames that contain matches\n\
                           (don't print the matching lines)\n\
   -L --files-without-matches\n\
                           Only print filenames that don't contain matches\n\
+     --print-all-files    Print headings for all files searched, even those that\n\
+                          don't contain matches\n\
      --[no]numbers        Print line numbers. Default is to omit line numbers\n\
                           when searching streams\n\
   -o --only-matching      Prints only the matching part of the lines\n\
@@ -70,6 +75,8 @@ Output Options:\n\
                           don't match\n\
      --silent             Suppress all log messages, including errors\n\
      --stats              Print stats (files scanned, time taken, etc.)\n\
+     --stats-only         Print stats and nothing else.\n\
+                          (Same as --count when searching a single file)\n\
      --vimgrep            Print results like vim's :vimgrep /pattern/g would\n\
                           (it reports every match on the line)\n\
   -0 --null --print0      Separate filenames with null (for 'xargs -0')\n\
@@ -89,20 +96,21 @@ Search Options:\n\
      --ignore-dir NAME    Alias for --ignore for compatibility with ack.\n\
   -m --max-count NUM      Skip the rest of a file after NUM matches (Default: 10,000)\n\
      --one-device         Don't follow links to other devices.\n\
-  -p --path-to-agignore STRING\n\
-                          Use .agignore file at STRING\n\
+  -p --path-to-ignore STRING\n\
+                          Use .ignore file at STRING\n\
   -Q --literal            Don't parse PATTERN as a regular expression\n\
   -s --case-sensitive     Match case sensitively\n\
   -S --smart-case         Match case insensitively unless PATTERN contains\n\
                           uppercase characters (Enabled by default)\n\
      --search-binary      Search binary files for matches\n\
   -t --all-text           Search all text files (doesn't include hidden files)\n\
-  -u --unrestricted       Search all files (ignore .agignore, .gitignore, etc.;\n\
+  -u --unrestricted       Search all files (ignore .ignore, .gitignore, etc.;\n\
                           searches binary and hidden files as well)\n\
   -U --skip-vcs-ignores   Ignore VCS ignore files\n\
-                          (.gitignore, .hgignore, .svnignore; still obey .agignore)\n\
+                          (.gitignore, .hgignore; still obey .ignore)\n\
   -v --invert-match\n\
   -w --word-regexp        Only match whole words\n\
+  -W --width NUM          Truncate match lines after NUM characters\n\
   -z --search-zip         Search contents of compressed (e.g., gzip) files\n\
 \n");
     printf("File Types:\n\
@@ -111,7 +119,9 @@ The search can be restricted to certain types of files. Example:\n\
   - Searches for 'needle' in files with suffix .htm, .html, .shtml or .xhtml.\n\
 \n\
 For a list of supported file types run:\n\
-  ag --list-file-types\n\n");
+  ag --list-file-types\n\n\
+ag was originally created by Geoff Greer. More information (and the latest release)\n\
+can be found at http://geoff.greer.fm/ag\n");
 }
 
 void print_version(void) {
@@ -135,16 +145,29 @@ void print_version(void) {
 }
 
 void init_options(void) {
+    char *term = getenv("TERM");
+
     memset(&opts, 0, sizeof(opts));
     opts.casing = CASE_DEFAULT;
     opts.color = TRUE;
+    if (term && !strcmp(term, "dumb")) {
+        opts.color = FALSE;
+    }
     opts.color_win_ansi = FALSE;
     opts.max_matches_per_file = 0;
     opts.max_search_depth = DEFAULT_MAX_SEARCH_DEPTH;
+#if defined(__APPLE__) || defined(__MACH__)
+    /* mamp() is slower than normal read() on macos. default to off */
+    opts.mmap = FALSE;
+#else
+    opts.mmap = TRUE;
+#endif
     opts.multiline = TRUE;
+    opts.width = 0;
     opts.path_sep = '\n';
     opts.print_break = TRUE;
     opts.print_path = PATH_PRINT_DEFAULT;
+    opts.print_all_paths = FALSE;
     opts.print_line_numbers = TRUE;
     opts.recurse_dirs = TRUE;
     opts.color_path = ag_strdup(color_path);
@@ -187,6 +210,7 @@ void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
     int ch;
     size_t i;
     int path_len = 0;
+    int base_path_len = 0;
     int useless = 0;
     int group = 1;
     int help = 0;
@@ -202,6 +226,7 @@ void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
     int rv;
     size_t lang_count;
     size_t lang_num = 0;
+    int has_filetype = 0;
 
     size_t longopts_len, full_len;
     option_t *longopts;
@@ -233,6 +258,7 @@ void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
         { "debug", no_argument, NULL, 'D' },
         { "depth", required_argument, NULL, 0 },
         { "filename", no_argument, NULL, 0 },
+        { "filename-pattern", required_argument, NULL, 'g' },
         { "file-search-regex", required_argument, NULL, 'G' },
         { "files-with-matches", no_argument, NULL, 'l' },
         { "files-without-matches", no_argument, NULL, 'L' },
@@ -252,20 +278,32 @@ void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
         { "literal", no_argument, NULL, 'Q' },
         { "match", no_argument, &useless, 0 },
         { "max-count", required_argument, NULL, 'm' },
+        { "mmap", no_argument, &opts.mmap, TRUE },
         { "multiline", no_argument, &opts.multiline, TRUE },
-        /* "no-" is deprecated. Remove these eventually. */
-        { "no-numbers", no_argument, &opts.print_line_numbers, FALSE },
-        { "no-recurse", no_argument, NULL, 'n' },
+        /* Accept both --no-* and --no* forms for convenience/BC */
+        { "no-affinity", no_argument, &opts.use_thread_affinity, 0 },
         { "noaffinity", no_argument, &opts.use_thread_affinity, 0 },
+        { "no-break", no_argument, &opts.print_break, 0 },
         { "nobreak", no_argument, &opts.print_break, 0 },
+        { "no-color", no_argument, &opts.color, 0 },
         { "nocolor", no_argument, &opts.color, 0 },
+        { "no-filename", no_argument, NULL, 0 },
         { "nofilename", no_argument, NULL, 0 },
+        { "no-follow", no_argument, &opts.follow_symlinks, 0 },
         { "nofollow", no_argument, &opts.follow_symlinks, 0 },
+        { "no-group", no_argument, &group, 0 },
         { "nogroup", no_argument, &group, 0 },
+        { "no-heading", no_argument, &opts.print_path, PATH_PRINT_EACH_LINE },
         { "noheading", no_argument, &opts.print_path, PATH_PRINT_EACH_LINE },
+        { "no-mmap", no_argument, &opts.mmap, FALSE },
+        { "nommap", no_argument, &opts.mmap, FALSE },
+        { "no-multiline", no_argument, &opts.multiline, FALSE },
         { "nomultiline", no_argument, &opts.multiline, FALSE },
+        { "no-numbers", no_argument, &opts.print_line_numbers, FALSE },
         { "nonumbers", no_argument, &opts.print_line_numbers, FALSE },
+        { "no-pager", no_argument, NULL, 0 },
         { "nopager", no_argument, NULL, 0 },
+        { "no-recurse", no_argument, NULL, 'n' },
         { "norecurse", no_argument, NULL, 'n' },
         { "null", no_argument, NULL, '0' },
         { "numbers", no_argument, &opts.print_line_numbers, 2 },
@@ -275,8 +313,9 @@ void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
         { "parallel", no_argument, &opts.parallel, 1 },
         { "passthrough", no_argument, &opts.passthrough, 1 },
         { "passthru", no_argument, &opts.passthrough, 1 },
-        { "path-to-agignore", required_argument, NULL, 'p' },
+        { "path-to-ignore", required_argument, NULL, 'p' },
         { "print0", no_argument, NULL, '0' },
+        { "print-all-files", no_argument, NULL, 0 },
         { "print-long-lines", no_argument, &opts.print_long_lines, 1 },
         { "recurse", no_argument, NULL, 'r' },
         { "search-binary", no_argument, &opts.search_binary_files, 1 },
@@ -286,9 +325,11 @@ void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
         { "skip-vcs-ignores", no_argument, NULL, 'U' },
         { "smart-case", no_argument, NULL, 'S' },
         { "stats", no_argument, &opts.stats, 1 },
+        { "stats-only", no_argument, NULL, 0 },
         { "unrestricted", no_argument, NULL, 'u' },
         { "version", no_argument, &version, 1 },
         { "vimgrep", no_argument, &opts.vimgrep, 1 },
+        { "width", required_argument, NULL, 'W' },
         { "word-regexp", no_argument, NULL, 'w' },
         { "workers", required_argument, NULL, 0 },
     };
@@ -337,8 +378,8 @@ void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
         opts.stdout_inode = statbuf.st_ino;
     }
 
-    int pcre_opts = 0;
-    while ((ch = getopt_long(argc, argv, "A:aB:C:cDG:g:FfHhiLlm:nop:QRrSsvVtuUwz0", longopts, &opt_index)) != -1) {
+    char *file_search_regex = NULL;
+    while ((ch = getopt_long(argc, argv, "A:aB:C:cDG:g:FfHhiLlm:nop:QRrSsvVtuUwW:z0", longopts, &opt_index)) != -1) {
         switch (ch) {
             case 'A':
                 if (optarg) {
@@ -393,15 +434,14 @@ void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
             case 'g':
                 needs_query = accepts_query = 0;
                 opts.match_files = 1;
-            /* Fall through and build regex */
+            /* fall through */
             case 'G':
-                if (opts.casing == CASE_DEFAULT) {
-                    opts.casing = CASE_SENSITIVE;
+                if (file_search_regex) {
+                    log_err("File search regex (-g or -G) already specified.");
+                    usage();
+                    exit(1);
                 }
-                if (opts.casing != CASE_SENSITIVE) {
-                    pcre_opts |= PCRE_CASELESS;
-                }
-                compile_study(&opts.file_search_regex, &opts.file_search_regex_extra, optarg, pcre_opts, 0);
+                file_search_regex = ag_strdup(optarg);
                 break;
             case 'H':
                 opts.print_path = PATH_PRINT_TOP;
@@ -413,11 +453,13 @@ void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
                 opts.casing = CASE_INSENSITIVE;
                 break;
             case 'L':
-                opts.invert_match = 1;
-            /* fall through */
+                opts.print_nonmatching_files = 1;
+                opts.print_path = PATH_PRINT_TOP;
+                break;
             case 'l':
                 needs_query = 0;
                 opts.print_filename_only = 1;
+                opts.print_path = PATH_PRINT_TOP;
                 break;
             case 'm':
                 opts.max_matches_per_file = atoi(optarg);
@@ -426,7 +468,8 @@ void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
                 opts.recurse_dirs = 0;
                 break;
             case 'p':
-                opts.path_to_agignore = optarg;
+                opts.path_to_ignore = TRUE;
+                load_ignore_patterns(root_ignores, optarg);
                 break;
             case 'o':
                 opts.only_matching = 1;
@@ -467,6 +510,12 @@ void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
             case 'w':
                 opts.word_regexp = 1;
                 break;
+            case 'W':
+                opts.width = strtol(optarg, &num_end, 10);
+                if (num_end == optarg || *num_end != '\0' || errno == ERANGE) {
+                    die("Invalid width\n");
+                }
+                break;
             case 'z':
                 opts.search_zip_files = 1;
                 break;
@@ -490,17 +539,22 @@ void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
                 } else if (strcmp(longopts[opt_index].name, "ignore") == 0) {
                     add_ignore_pattern(root_ignores, optarg);
                     break;
-                } else if (strcmp(longopts[opt_index].name, "nofilename") == 0) {
+                } else if (strcmp(longopts[opt_index].name, "no-filename") == 0 ||
+                           strcmp(longopts[opt_index].name, "nofilename") == 0) {
                     opts.print_path = PATH_PRINT_NOTHING;
                     opts.print_line_numbers = FALSE;
                     break;
-                } else if (strcmp(longopts[opt_index].name, "nopager") == 0) {
+                } else if (strcmp(longopts[opt_index].name, "no-pager") == 0 ||
+                           strcmp(longopts[opt_index].name, "nopager") == 0) {
                     out_fd = stdout;
                     opts.pager = NULL;
                     break;
                 } else if (strcmp(longopts[opt_index].name, "pager") == 0) {
                     opts.pager = optarg;
                     break;
+                } else if (strcmp(longopts[opt_index].name, "print-all-files") == 0) {
+                    opts.print_all_paths = TRUE;
+                    break;
                 } else if (strcmp(longopts[opt_index].name, "workers") == 0) {
                     opts.workers = atoi(optarg);
                     break;
@@ -519,6 +573,11 @@ void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
                 } else if (strcmp(longopts[opt_index].name, "silent") == 0) {
                     set_log_level(LOG_LEVEL_NONE);
                     break;
+                } else if (strcmp(longopts[opt_index].name, "stats-only") == 0) {
+                    opts.print_filename_only = 1;
+                    opts.print_path = PATH_PRINT_NOTHING;
+                    opts.stats = 1;
+                    break;
                 }
 
                 /* Continue to usage if we don't recognize the option */
@@ -528,6 +587,7 @@ void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
 
                 for (i = 0; i < lang_count; i++) {
                     if (strcmp(longopts[opt_index].name, langs[i].name) == 0) {
+                        has_filetype = 1;
                         ext_index[lang_num++] = i;
                         break;
                     }
@@ -537,14 +597,32 @@ void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
                 }
 
                 log_err("option %s does not take a value", longopts[opt_index].name);
+            /* fall through */
             default:
                 usage();
                 exit(1);
         }
     }
 
+    if (opts.casing == CASE_DEFAULT) {
+        opts.casing = CASE_SMART;
+    }
 
-    if (ext_index[0]) {
+    if (file_search_regex) {
+        int pcre_opts = 0;
+        if (opts.casing == CASE_INSENSITIVE || (opts.casing == CASE_SMART && is_lowercase(file_search_regex))) {
+            pcre_opts |= PCRE_CASELESS;
+        }
+        if (opts.word_regexp) {
+            char *old_file_search_regex = file_search_regex;
+            ag_asprintf(&file_search_regex, "\\b%s\\b", file_search_regex);
+            free(old_file_search_regex);
+        }
+        compile_study(&opts.file_search_regex, &opts.file_search_regex_extra, file_search_regex, pcre_opts, 0);
+        free(file_search_regex);
+    }
+
+    if (has_filetype) {
         num_exts = combine_file_extensions(ext_index, lang_num, &extensions);
         lang_regex = make_lang_regex(extensions, num_exts);
         compile_study(&opts.file_search_regex, &opts.file_search_regex_extra, lang_regex, 0, 0);
@@ -562,10 +640,6 @@ void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
     argc -= optind;
     argv += optind;
 
-    if (opts.casing == CASE_DEFAULT) {
-        opts.casing = CASE_SMART;
-    }
-
     if (opts.pager) {
         out_fd = popen(opts.pager, "w");
         if (!out_fd) {
@@ -574,6 +648,14 @@ void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
         }
     }
 
+#ifdef HAVE_PLEDGE
+    if (opts.skip_vcs_ignores) {
+        if (pledge("stdio rpath proc", NULL) == -1) {
+            die("pledge: %s", strerror(errno));
+        }
+    }
+#endif
+
     if (help) {
         usage();
         exit(0);
@@ -605,7 +687,7 @@ void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
 
     if (home_dir && !opts.search_all_files) {
         log_debug("Found user's home dir: %s", home_dir);
-        ag_asprintf(&ignore_file_path, "%s/%s", home_dir, ignore_pattern_files[0]);
+        ag_asprintf(&ignore_file_path, "%s/.agignore", home_dir);
         load_ignore_patterns(root_ignores, ignore_file_path);
         free(ignore_file_path);
     }
@@ -626,13 +708,30 @@ void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
                 buf_len += fread(gitconfig_res + buf_len, 1, 64, gitconfig_file);
             } while (!feof(gitconfig_file) && buf_len > 0 && buf_len % 64 == 0);
             gitconfig_res[buf_len] = '\0';
-            log_debug("Found user's global Git excludesfile: %s", gitconfig_res);
+            if (buf_len == 0) {
+                free(gitconfig_res);
+                const char *config_home = getenv("XDG_CONFIG_HOME");
+                if (config_home) {
+                    ag_asprintf(&gitconfig_res, "%s/%s", config_home, "git/ignore");
+                } else if (home_dir) {
+                    ag_asprintf(&gitconfig_res, "%s/%s", home_dir, ".config/git/ignore");
+                } else {
+                    gitconfig_res = ag_strdup("");
+                }
+            }
+            log_debug("global core.excludesfile: %s", gitconfig_res);
             load_ignore_patterns(root_ignores, gitconfig_res);
             free(gitconfig_res);
             pclose(gitconfig_file);
         }
     }
 
+#ifdef HAVE_PLEDGE
+    if (pledge("stdio rpath proc", NULL) == -1) {
+        die("pledge: %s", strerror(errno));
+    }
+#endif
+
     if (opts.context > 0) {
         opts.before = opts.context;
         opts.after = opts.context;
@@ -675,8 +774,13 @@ void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
     }
 
     if (accepts_query && argc > 0) {
-        // use the provided query
-        opts.query = ag_strdup(argv[0]);
+        if (!needs_query && strlen(argv[0]) == 0) {
+            // use default query
+            opts.query = ag_strdup(".");
+        } else {
+            // use the provided query
+            opts.query = ag_strdup(argv[0]);
+        }
         argc--;
         argv++;
     } else if (!needs_query) {
@@ -697,7 +801,10 @@ void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
     }
 
     char *path = NULL;
+    char *base_path = NULL;
+#ifdef PATH_MAX
     char *tmp = NULL;
+#endif
     opts.paths_len = argc;
     if (argc > 0) {
         *paths = ag_calloc(sizeof(char *), argc + 1);
@@ -710,8 +817,22 @@ void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
                 path[path_len - 1] = '\0';
             }
             (*paths)[i] = path;
+#ifdef PATH_MAX
             tmp = ag_malloc(PATH_MAX);
-            (*base_paths)[i] = realpath(path, tmp);
+            base_path = realpath(path, tmp);
+#else
+            base_path = realpath(path, NULL);
+#endif
+            if (base_path) {
+                base_path_len = strlen(base_path);
+                /* add trailing slash */
+                if (base_path_len > 1 && base_path[base_path_len - 1] != '/') {
+                    base_path = ag_realloc(base_path, base_path_len + 2);
+                    base_path[base_path_len] = '/';
+                    base_path[base_path_len + 1] = '\0';
+                }
+            }
+            (*base_paths)[i] = base_path;
         }
         /* Make sure we search these paths instead of stdin. */
         opts.search_stream = 0;
@@ -720,8 +841,12 @@ void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
         *paths = ag_malloc(sizeof(char *) * 2);
         *base_paths = ag_malloc(sizeof(char *) * 2);
         (*paths)[0] = path;
+#ifdef PATH_MAX
         tmp = ag_malloc(PATH_MAX);
         (*base_paths)[0] = realpath(path, tmp);
+#else
+        (*base_paths)[0] = realpath(path, NULL);
+#endif
         i = 1;
     }
     (*paths)[i] = NULL;

src/options.h

@@ -51,15 +51,18 @@ typedef struct {
     int literal_ends_wordchar;
     size_t max_matches_per_file;
     int max_search_depth;
+    int mmap;
     int multiline;
     int one_dev;
     int only_matching;
     char path_sep;
-    char *path_to_agignore;
+    int path_to_ignore;
     int print_break;
     int print_count;
     int print_filename_only;
+    int print_nonmatching_files;
     int print_path;
+    int print_all_paths;
     int print_line_numbers;
     int print_long_lines; /* TODO: support this in print.c */
     int passthrough;
@@ -83,12 +86,13 @@ typedef struct {
     int parallel;
     int use_thread_affinity;
     int vimgrep;
+    size_t width;
     int word_regexp;
     int workers;
 } cli_options;
 
 /* global options. parse_options gives it sane values, everything else reads from it */
-cli_options opts;
+extern cli_options opts;
 
 typedef struct option option_t;
 

src/print.c

@@ -1,13 +1,14 @@
+#include <limits.h>
 #include <stdarg.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <limits.h>
 
 #include "ignore.h"
 #include "log.h"
 #include "options.h"
 #include "print.h"
+#include "search.h"
 #include "util.h"
 #ifdef _WIN32
 #define fprintf(...) fprintf_w32(__VA_ARGS__)
@@ -17,6 +18,88 @@ int first_file_match = 1;
 
 const char *color_reset = "\033[0m\033[K";
 
+const char *truncate_marker = " [...]";
+
+__thread struct print_context {
+    size_t line;
+    char **context_prev_lines;
+    size_t prev_line;
+    size_t last_prev_line;
+    size_t prev_line_offset;
+    size_t line_preceding_current_match_offset;
+    size_t lines_since_last_match;
+    size_t last_printed_match;
+    int in_a_match;
+    int printing_a_match;
+} print_context;
+
+void print_init_context(void) {
+    if (print_context.context_prev_lines != NULL) {
+        return;
+    }
+    print_context.context_prev_lines = ag_calloc(sizeof(char *), (opts.before + 1));
+    print_context.line = 1;
+    print_context.prev_line = 0;
+    print_context.last_prev_line = 0;
+    print_context.prev_line_offset = 0;
+    print_context.line_preceding_current_match_offset = 0;
+    print_context.lines_since_last_match = INT_MAX;
+    print_context.last_printed_match = 0;
+    print_context.in_a_match = FALSE;
+    print_context.printing_a_match = FALSE;
+}
+
+void print_cleanup_context(void) {
+    size_t i;
+
+    if (print_context.context_prev_lines == NULL) {
+        return;
+    }
+
+    for (i = 0; i < opts.before; i++) {
+        if (print_context.context_prev_lines[i] != NULL) {
+            free(print_context.context_prev_lines[i]);
+        }
+    }
+    free(print_context.context_prev_lines);
+    print_context.context_prev_lines = NULL;
+}
+
+void print_context_append(const char *line, size_t len) {
+    if (opts.before == 0) {
+        return;
+    }
+    if (print_context.context_prev_lines[print_context.last_prev_line] != NULL) {
+        free(print_context.context_prev_lines[print_context.last_prev_line]);
+    }
+    print_context.context_prev_lines[print_context.last_prev_line] = ag_strndup(line, len);
+    print_context.last_prev_line = (print_context.last_prev_line + 1) % opts.before;
+}
+
+void print_trailing_context(const char *path, const char *buf, size_t n) {
+    char sep = '-';
+
+    if (opts.ackmate || opts.vimgrep) {
+        sep = ':';
+    }
+
+    if (print_context.lines_since_last_match != 0 &&
+        print_context.lines_since_last_match <= opts.after) {
+        if (opts.print_path == PATH_PRINT_EACH_LINE) {
+            print_path(path, ':');
+        }
+        print_line_number(print_context.line, sep);
+
+        fwrite(buf, 1, n, out_fd);
+        fputc('\n', out_fd);
+    }
+
+    print_context.line++;
+    if (!print_context.in_a_match && print_context.lines_since_last_match < INT_MAX) {
+        print_context.lines_since_last_match++;
+    }
+}
+
 void print_path(const char *path, const char sep) {
     if (opts.print_path == PATH_PRINT_NOTHING && !opts.vimgrep) {
         return;
@@ -48,7 +131,12 @@ void print_path_count(const char *path, const char sep, const size_t count) {
 }
 
 void print_line(const char *buf, size_t buf_pos, size_t prev_line_offset) {
-    fwrite(buf + prev_line_offset, 1, buf_pos - prev_line_offset + 1, out_fd);
+    size_t write_chars = buf_pos - prev_line_offset + 1;
+    if (opts.width > 0 && opts.width < write_chars) {
+        write_chars = opts.width;
+    }
+
+    fwrite(buf + prev_line_offset, 1, write_chars, out_fd);
 }
 
 void print_binary_file_matches(const char *path) {
@@ -58,19 +146,11 @@ void print_binary_file_matches(const char *path) {
 }
 
 void print_file_matches(const char *path, const char *buf, const size_t buf_len, const match_t matches[], const size_t matches_len) {
-    size_t line = 1;
-    char **context_prev_lines = NULL;
-    size_t prev_line = 0;
-    size_t last_prev_line = 0;
-    size_t prev_line_offset = 0;
     size_t cur_match = 0;
-    size_t lines_since_last_match = INT_MAX;
     ssize_t lines_to_print = 0;
-    size_t last_printed_match = 0;
     char sep = '-';
     size_t i, j;
-    int in_a_match = FALSE;
-    int printing_a_match = FALSE;
+    int blanks_between_matches = opts.context || opts.after || opts.before;
 
     if (opts.ackmate || opts.vimgrep) {
         sep = ':';
@@ -92,20 +172,18 @@ void print_file_matches(const char *path, const char *buf, const size_t buf_len,
         }
     }
 
-    context_prev_lines = ag_calloc(sizeof(char *), (opts.before + 1));
-
-    for (i = 0; i <= buf_len && (cur_match < matches_len || lines_since_last_match <= opts.after); i++) {
+    for (i = 0; i <= buf_len && (cur_match < matches_len || print_context.lines_since_last_match <= opts.after); i++) {
         if (cur_match < matches_len && i == matches[cur_match].start) {
-            in_a_match = TRUE;
+            print_context.in_a_match = TRUE;
             /* We found the start of a match */
-            if (cur_match > 0 && opts.context && lines_since_last_match > (opts.before + opts.after + 1)) {
+            if (cur_match > 0 && blanks_between_matches && print_context.lines_since_last_match > (opts.before + opts.after + 1)) {
                 fprintf(out_fd, "--\n");
             }
 
-            if (lines_since_last_match > 0 && opts.before > 0) {
+            if (print_context.lines_since_last_match > 0 && opts.before > 0) {
                 /* TODO: better, but still needs work */
                 /* print the previous line(s) */
-                lines_to_print = lines_since_last_match - (opts.after + 1);
+                lines_to_print = print_context.lines_since_last_match - (opts.after + 1);
                 if (lines_to_print < 0) {
                     lines_to_print = 0;
                 } else if ((size_t)lines_to_print > opts.before) {
@@ -113,151 +191,150 @@ void print_file_matches(const char *path, const char *buf, const size_t buf_len,
                 }
 
                 for (j = (opts.before - lines_to_print); j < opts.before; j++) {
-                    prev_line = (last_prev_line + j) % opts.before;
-                    if (context_prev_lines[prev_line] != NULL) {
+                    print_context.prev_line = (print_context.last_prev_line + j) % opts.before;
+                    if (print_context.context_prev_lines[print_context.prev_line] != NULL) {
                         if (opts.print_path == PATH_PRINT_EACH_LINE) {
                             print_path(path, ':');
                         }
-                        print_line_number(line - (opts.before - j), sep);
-                        fprintf(out_fd, "%s\n", context_prev_lines[prev_line]);
+                        print_line_number(print_context.line - (opts.before - j), sep);
+                        fprintf(out_fd, "%s\n", print_context.context_prev_lines[print_context.prev_line]);
                     }
                 }
             }
-            lines_since_last_match = 0;
+            print_context.lines_since_last_match = 0;
         }
 
         if (cur_match < matches_len && i == matches[cur_match].end) {
             /* We found the end of a match. */
             cur_match++;
-            in_a_match = FALSE;
+            print_context.in_a_match = FALSE;
         }
 
         /* We found the end of a line. */
-        if (buf[i] == '\n' && opts.before > 0) {
-            if (context_prev_lines[last_prev_line] != NULL) {
-                free(context_prev_lines[last_prev_line]);
-            }
+        if ((i == buf_len || buf[i] == '\n') && opts.before > 0) {
             /* We don't want to strcpy the \n */
-            context_prev_lines[last_prev_line] = ag_strndup(&buf[prev_line_offset], i - prev_line_offset);
-            last_prev_line = (last_prev_line + 1) % opts.before;
+            print_context_append(&buf[print_context.prev_line_offset], i - print_context.prev_line_offset);
         }
 
-        if (buf[i] == '\n' || i == buf_len) {
-            if (lines_since_last_match == 0) {
+        if (i == buf_len || buf[i] == '\n') {
+            if (print_context.lines_since_last_match == 0) {
                 if (opts.print_path == PATH_PRINT_EACH_LINE && !opts.search_stream) {
                     print_path(path, ':');
                 }
                 if (opts.ackmate) {
                     /* print headers for ackmate to parse */
-                    print_line_number(line, ';');
-                    for (; last_printed_match < cur_match; last_printed_match++) {
-                        /* Don't print negative offsets. This isn't quite right, but not many people use --ackmate */
-                        long start = (long)(matches[last_printed_match].start - prev_line_offset);
-                        if (start < 0) {
-                            start = 0;
-                        }
-                        fprintf(out_fd, "%li %li",
+                    print_line_number(print_context.line, ';');
+                    for (; print_context.last_printed_match < cur_match; print_context.last_printed_match++) {
+                        size_t start = matches[print_context.last_printed_match].start - print_context.line_preceding_current_match_offset;
+                        fprintf(out_fd, "%lu %lu",
                                 start,
-                                (long)(matches[last_printed_match].end - matches[last_printed_match].start));
-                        last_printed_match == cur_match - 1 ? fputc(':', out_fd) : fputc(',', out_fd);
+                                matches[print_context.last_printed_match].end - matches[print_context.last_printed_match].start);
+                        print_context.last_printed_match == cur_match - 1 ? fputc(':', out_fd) : fputc(',', out_fd);
                     }
-                    print_line(buf, i, prev_line_offset);
+                    print_line(buf, i, print_context.prev_line_offset);
                 } else if (opts.vimgrep) {
-                    for (; last_printed_match < cur_match; last_printed_match++) {
+                    for (; print_context.last_printed_match < cur_match; print_context.last_printed_match++) {
                         print_path(path, sep);
-                        print_line_number(line, sep);
-                        print_column_number(matches, last_printed_match, prev_line_offset, sep);
-                        print_line(buf, i, prev_line_offset);
+                        print_line_number(print_context.line, sep);
+                        print_column_number(matches, print_context.last_printed_match, print_context.prev_line_offset, sep);
+                        print_line(buf, i, print_context.prev_line_offset);
                     }
                 } else {
-                    print_line_number(line, ':');
+                    print_line_number(print_context.line, ':');
                     int printed_match = FALSE;
                     if (opts.column) {
-                        print_column_number(matches, last_printed_match, prev_line_offset, ':');
+                        print_column_number(matches, print_context.last_printed_match, print_context.prev_line_offset, ':');
                     }
 
-                    if (printing_a_match && opts.color) {
+                    if (print_context.printing_a_match && opts.color) {
                         fprintf(out_fd, "%s", opts.color_match);
                     }
-                    for (j = prev_line_offset; j <= i; j++) {
-                        if (last_printed_match < matches_len && j == matches[last_printed_match].end) {
+                    for (j = print_context.prev_line_offset; j <= i; j++) {
+                        /* close highlight of match term */
+                        if (print_context.last_printed_match < matches_len && j == matches[print_context.last_printed_match].end) {
                             if (opts.color) {
                                 fprintf(out_fd, "%s", color_reset);
                             }
-                            printing_a_match = FALSE;
-                            last_printed_match++;
+                            print_context.printing_a_match = FALSE;
+                            print_context.last_printed_match++;
                             printed_match = TRUE;
                             if (opts.only_matching) {
                                 fputc('\n', out_fd);
                             }
                         }
-                        if (last_printed_match < matches_len && j == matches[last_printed_match].start) {
+                        /* skip remaining characters if truncation width exceeded, needs to be done
+                         * before highlight opening */
+                        if (j < buf_len && opts.width > 0 && j - print_context.prev_line_offset >= opts.width) {
+                            if (j < i) {
+                                fputs(truncate_marker, out_fd);
+                            }
+                            fputc('\n', out_fd);
+
+                            /* prevent any more characters or highlights */
+                            j = i;
+                            print_context.last_printed_match = matches_len;
+                        }
+                        /* open highlight of match term */
+                        if (print_context.last_printed_match < matches_len && j == matches[print_context.last_printed_match].start) {
                             if (opts.only_matching && printed_match) {
                                 if (opts.print_path == PATH_PRINT_EACH_LINE) {
                                     print_path(path, ':');
                                 }
-                                print_line_number(line, ':');
+                                print_line_number(print_context.line, ':');
                                 if (opts.column) {
-                                    print_column_number(matches, last_printed_match, prev_line_offset, ':');
+                                    print_column_number(matches, print_context.last_printed_match, print_context.prev_line_offset, ':');
                                 }
                             }
                             if (opts.color) {
                                 fprintf(out_fd, "%s", opts.color_match);
                             }
-                            printing_a_match = TRUE;
+                            print_context.printing_a_match = TRUE;
                         }
                         /* Don't print the null terminator */
                         if (j < buf_len) {
                             /* if only_matching is set, print only matches and newlines */
-                            if (!opts.only_matching || printing_a_match) {
-                                fputc(buf[j], out_fd);
+                            if (!opts.only_matching || print_context.printing_a_match) {
+                                if (opts.width == 0 || j - print_context.prev_line_offset < opts.width) {
+                                    fputc(buf[j], out_fd);
+                                }
                             }
                         }
                     }
-                    if (printing_a_match && opts.color) {
+                    if (print_context.printing_a_match && opts.color) {
                         fprintf(out_fd, "%s", color_reset);
                     }
                 }
-            } else if (lines_since_last_match <= opts.after) {
-                /* print context after matching line */
-                if (opts.print_path == PATH_PRINT_EACH_LINE) {
-                    print_path(path, ':');
-                }
-                print_line_number(line, sep);
-
-                for (j = prev_line_offset; j < i; j++) {
-                    fputc(buf[j], out_fd);
-                }
-                fputc('\n', out_fd);
             }
 
-            prev_line_offset = i + 1; /* skip the newline */
-            line++;
-            if (!in_a_match && lines_since_last_match < INT_MAX) {
-                lines_since_last_match++;
+            if (opts.search_stream) {
+                print_context.last_printed_match = 0;
+                break;
             }
+
+            /* print context after matching line */
+            print_trailing_context(path, &buf[print_context.prev_line_offset], i - print_context.prev_line_offset);
+
+            print_context.prev_line_offset = i + 1; /* skip the newline */
+            if (!print_context.in_a_match) {
+                print_context.line_preceding_current_match_offset = i + 1;
+            }
+
             /* File doesn't end with a newline. Print one so the output is pretty. */
-            if (i == buf_len && buf[i] != '\n' && !opts.search_stream) {
+            if (i == buf_len && buf[i - 1] != '\n') {
                 fputc('\n', out_fd);
             }
         }
     }
-
-    for (i = 0; i < opts.before; i++) {
-        if (context_prev_lines[i] != NULL) {
-            free(context_prev_lines[i]);
-        }
+    /* Flush output if stdout is not a tty */
+    if (opts.stdout_inode) {
+        fflush(out_fd);
     }
-    free(context_prev_lines);
 }
 
 void print_line_number(size_t line, const char sep) {
     if (!opts.print_line_numbers) {
         return;
     }
-    if (opts.search_stream && opts.stream_line_num) {
-        line = opts.stream_line_num;
-    }
     if (opts.color) {
         fprintf(out_fd, "%s%lu%s%c", opts.color_line_number, (unsigned long)line, color_reset, sep);
     } else {

src/print.h

@@ -3,6 +3,10 @@
 
 #include "util.h"
 
+void print_init_context(void);
+void print_cleanup_context(void);
+void print_context_append(const char *line, size_t len);
+void print_trailing_context(const char *path, const char *buf, size_t n);
 void print_path(const char *path, const char sep);
 void print_path_count(const char *path, const char sep, const size_t count);
 void print_line(const char *buf, size_t buf_pos, size_t prev_line_offset);

src/print_w32.c

@@ -1,10 +1,10 @@
 #ifdef _WIN32
 
-#include <windows.h>
-#include <stdio.h>
+#include "print.h"
 #include <io.h>
 #include <stdarg.h>
-#include "print.h"
+#include <stdio.h>
+#include <windows.h>
 
 #ifndef FOREGROUND_MASK
 #define FOREGROUND_MASK (FOREGROUND_RED | FOREGROUND_BLUE | FOREGROUND_GREEN | FOREGROUND_INTENSITY)

src/scandir.h

@@ -7,6 +7,7 @@ typedef struct {
     const ignores *ig;
     const char *base_path;
     size_t base_path_len;
+    const char *path_start;
 } scandir_baton_t;
 
 typedef int (*filter_fp)(const char *path, const struct dirent *, void *);

src/search.c

@@ -1,18 +1,33 @@
 #include "search.h"
+#include "print.h"
 #include "scandir.h"
 
-void search_buf(const char *buf, const size_t buf_len,
-                const char *dir_full_path) {
+size_t alpha_skip_lookup[256];
+size_t *find_skip_lookup;
+uint8_t h_table[H_SIZE] __attribute__((aligned(64)));
+
+work_queue_t *work_queue = NULL;
+work_queue_t *work_queue_tail = NULL;
+int done_adding_files = 0;
+pthread_cond_t files_ready = PTHREAD_COND_INITIALIZER;
+pthread_mutex_t stats_mtx = PTHREAD_MUTEX_INITIALIZER;
+pthread_mutex_t work_queue_mtx = PTHREAD_MUTEX_INITIALIZER;
+
+symdir_t *symhash = NULL;
+
+/* Returns: -1 if skipped, otherwise # of matches */
+ssize_t search_buf(const char *buf, const size_t buf_len,
+                   const char *dir_full_path) {
     int binary = -1; /* 1 = yes, 0 = no, -1 = don't know */
     size_t buf_offset = 0;
 
     if (opts.search_stream) {
         binary = 0;
-    } else if (!opts.search_binary_files) {
+    } else if (!opts.search_binary_files && opts.mmap) { /* if not using mmap, binary files have already been skipped */
         binary = is_binary((const void *)buf, buf_len);
         if (binary) {
             log_debug("File %s is binary. Skipping...", dir_full_path);
-            return;
+            return -1;
         }
     }
 
@@ -38,16 +53,26 @@ void search_buf(const char *buf, const size_t buf_len,
 
     if (!opts.literal && opts.query_len == 1 && opts.query[0] == '.') {
         matches_size = 1;
-        matches = ag_malloc(matches_size * sizeof(match_t));
+        matches = matches == NULL ? ag_malloc(matches_size * sizeof(match_t)) : matches;
         matches[0].start = 0;
         matches[0].end = buf_len;
         matches_len = 1;
     } else if (opts.literal) {
         const char *match_ptr = buf;
-        strncmp_fp ag_strnstr_fp = get_strstr(opts.casing);
 
         while (buf_offset < buf_len) {
-            match_ptr = ag_strnstr_fp(match_ptr, opts.query, buf_len - buf_offset, opts.query_len, alpha_skip_lookup, find_skip_lookup);
+/* hash_strnstr only for little-endian platforms that allow unaligned access */
+#if defined(__i386__) || defined(__x86_64__)
+            /* Decide whether to fall back on boyer-moore */
+            if ((size_t)opts.query_len < 2 * sizeof(uint16_t) - 1 || opts.query_len >= UCHAR_MAX) {
+                match_ptr = boyer_moore_strnstr(match_ptr, opts.query, buf_len - buf_offset, opts.query_len, alpha_skip_lookup, find_skip_lookup, opts.casing == CASE_INSENSITIVE);
+            } else {
+                match_ptr = hash_strnstr(match_ptr, opts.query, buf_len - buf_offset, opts.query_len, h_table, opts.casing == CASE_SENSITIVE);
+            }
+#else
+            match_ptr = boyer_moore_strnstr(match_ptr, opts.query, buf_len - buf_offset, opts.query_len, alpha_skip_lookup, find_skip_lookup, opts.casing == CASE_INSENSITIVE);
+#endif
+
             if (match_ptr == NULL) {
                 break;
             }
@@ -66,8 +91,8 @@ void search_buf(const char *buf, const size_t buf_len,
                     /* It's a match */
                 } else {
                     /* It's not a match */
-                    match_ptr += opts.query_len;
-                    buf_offset = end - buf;
+                    match_ptr += find_skip_lookup[0] - opts.query_len + 1;
+                    buf_offset = match_ptr - buf;
                     continue;
                 }
             }
@@ -111,7 +136,7 @@ void search_buf(const char *buf, const size_t buf_len,
             }
         } else {
             while (buf_offset < buf_len) {
-                char *line;
+                const char *line;
                 size_t line_len = buf_getline(&line, buf, buf_len, buf_offset);
                 if (!line) {
                     break;
@@ -163,25 +188,16 @@ multiline_done:
         pthread_mutex_unlock(&stats_mtx);
     }
 
-    if (matches_len > 0) {
+    if (!opts.print_nonmatching_files && (matches_len > 0 || opts.print_all_paths)) {
         if (binary == -1 && !opts.print_filename_only) {
             binary = is_binary((const void *)buf, buf_len);
         }
         pthread_mutex_lock(&print_mtx);
         if (opts.print_filename_only) {
-            /* If the --files-without-matches or -L option is passed we should
-             * not print a matching line. This option currently sets
-             * opts.print_filename_only and opts.invert_match. Unfortunately
-             * setting the latter has the side effect of making matches.len = 1
-             * on a file-without-matches which is not desired behaviour. See
-             * GitHub issue 206 for the consequences if this behaviour is not
-             * checked. */
-            if (!opts.invert_match || matches_len < 2) {
-                if (opts.print_count) {
-                    print_path_count(dir_full_path, opts.path_sep, (size_t)matches_len);
-                } else {
-                    print_path(dir_full_path, opts.path_sep);
-                }
+            if (opts.print_count) {
+                print_path_count(dir_full_path, opts.path_sep, (size_t)matches_len);
+            } else {
+                print_path(dir_full_path, opts.path_sep);
             }
         } else if (binary) {
             print_binary_file_matches(dir_full_path);
@@ -196,33 +212,77 @@ multiline_done:
         log_debug("No match in %s", dir_full_path);
     }
 
+    if (matches_len == 0 && opts.search_stream) {
+        print_context_append(buf, buf_len - 1);
+    }
+
     if (matches_size > 0) {
         free(matches);
     }
+
+    /* FIXME: handle case where matches_len > SSIZE_MAX */
+    return (ssize_t)matches_len;
 }
 
+/* Return value: -1 if skipped, otherwise # of matches */
 /* TODO: this will only match single lines. multi-line regexes silently don't match */
-void search_stream(FILE *stream, const char *path) {
+ssize_t search_stream(FILE *stream, const char *path) {
     char *line = NULL;
+    ssize_t matches_count = 0;
     ssize_t line_len = 0;
     size_t line_cap = 0;
     size_t i;
 
+    print_init_context();
+
     for (i = 1; (line_len = getline(&line, &line_cap, stream)) > 0; i++) {
+        ssize_t result;
         opts.stream_line_num = i;
-        search_buf(line, line_len, path);
+        result = search_buf(line, line_len, path);
+        if (result > 0) {
+            if (matches_count == -1) {
+                matches_count = 0;
+            }
+            matches_count += result;
+        } else if (matches_count <= 0 && result == -1) {
+            matches_count = -1;
+        }
+        if (line[line_len - 1] == '\n') {
+            line_len--;
+        }
+        print_trailing_context(path, line, line_len);
     }
 
     free(line);
+    print_cleanup_context();
+    return matches_count;
 }
 
 void search_file(const char *file_full_path) {
-    int fd;
+    int fd = -1;
     off_t f_len = 0;
     char *buf = NULL;
     struct stat statbuf;
     int rv = 0;
-    FILE *pipe = NULL;
+    int matches_count = -1;
+    FILE *fp = NULL;
+
+    rv = stat(file_full_path, &statbuf);
+    if (rv != 0) {
+        log_err("Skipping %s: Error fstat()ing file.", file_full_path);
+        goto cleanup;
+    }
+
+    if (opts.stdout_inode != 0 && opts.stdout_inode == statbuf.st_ino) {
+        log_debug("Skipping %s: stdout is redirected to it", file_full_path);
+        goto cleanup;
+    }
+
+    // handling only regular files and FIFOs
+    if (!S_ISREG(statbuf.st_mode) && !S_ISFIFO(statbuf.st_mode)) {
+        log_err("Skipping %s: Mode %u is not a file.", file_full_path, statbuf.st_mode);
+        goto cleanup;
+    }
 
     fd = open(file_full_path, O_RDONLY);
     if (fd < 0) {
@@ -231,6 +291,7 @@ void search_file(const char *file_full_path) {
         goto cleanup;
     }
 
+    // repeating stat check with file handle to prevent TOCTOU issue
     rv = fstat(fd, &statbuf);
     if (rv != 0) {
         log_err("Skipping %s: Error fstat()ing file.", file_full_path);
@@ -242,23 +303,30 @@ void search_file(const char *file_full_path) {
         goto cleanup;
     }
 
-    if ((statbuf.st_mode & S_IFMT) == 0) {
+    // handling only regular files and FIFOs
+    if (!S_ISREG(statbuf.st_mode) && !S_ISFIFO(statbuf.st_mode)) {
         log_err("Skipping %s: Mode %u is not a file.", file_full_path, statbuf.st_mode);
         goto cleanup;
     }
 
+    print_init_context();
+
     if (statbuf.st_mode & S_IFIFO) {
         log_debug("%s is a named pipe. stream searching", file_full_path);
-        pipe = fdopen(fd, "r");
-        search_stream(pipe, file_full_path);
-        fclose(pipe);
+        fp = fdopen(fd, "r");
+        matches_count = search_stream(fp, file_full_path);
+        fclose(fp);
         goto cleanup;
     }
 
     f_len = statbuf.st_size;
 
     if (f_len == 0) {
-        log_debug("Skipping %s: file is empty.", file_full_path);
+        if (opts.query[0] == '.' && opts.query_len == 1 && !opts.literal && opts.search_all_files) {
+            matches_count = search_buf(buf, f_len, file_full_path);
+        } else {
+            log_debug("Skipping %s: file is empty.", file_full_path);
+        }
         goto cleanup;
     }
 
@@ -286,42 +354,86 @@ void search_file(const char *file_full_path) {
         goto cleanup;
     }
 #else
-    buf = mmap(0, f_len, PROT_READ, MAP_SHARED, fd, 0);
-    if (buf == MAP_FAILED) {
-        log_err("File %s failed to load: %s.", file_full_path, strerror(errno));
-        goto cleanup;
-    }
+
+    if (opts.mmap) {
+        buf = mmap(0, f_len, PROT_READ, MAP_PRIVATE, fd, 0);
+        if (buf == MAP_FAILED) {
+            log_err("File %s failed to load: %s.", file_full_path, strerror(errno));
+            goto cleanup;
+        }
 #if HAVE_MADVISE
-    madvise(buf, f_len, MADV_SEQUENTIAL);
+        madvise(buf, f_len, MADV_SEQUENTIAL);
 #elif HAVE_POSIX_FADVISE
-    posix_fadvise(fd, 0, f_len, POSIX_MADV_SEQUENTIAL);
+        posix_fadvise(fd, 0, f_len, POSIX_MADV_SEQUENTIAL);
 #endif
+    } else {
+        buf = ag_malloc(f_len);
+
+        ssize_t bytes_read = 0;
+
+        if (!opts.search_binary_files) {
+            bytes_read += read(fd, buf, ag_min(f_len, 512));
+            // Optimization: If skipping binary files, don't read the whole buffer before checking if binary or not.
+            if (is_binary(buf, f_len)) {
+                log_debug("File %s is binary. Skipping...", file_full_path);
+                goto cleanup;
+            }
+        }
+
+        while (bytes_read < f_len) {
+            bytes_read += read(fd, buf + bytes_read, f_len);
+        }
+        if (bytes_read != f_len) {
+            die("File %s read(): expected to read %u bytes but read %u", file_full_path, f_len, bytes_read);
+        }
+    }
 #endif
 
     if (opts.search_zip_files) {
         ag_compression_type zip_type = is_zipped(buf, f_len);
         if (zip_type != AG_NO_COMPRESSION) {
+#if HAVE_FOPENCOOKIE
+            log_debug("%s is a compressed file. stream searching", file_full_path);
+            fp = decompress_open(fd, "r", zip_type);
+            matches_count = search_stream(fp, file_full_path);
+            fclose(fp);
+#else
             int _buf_len = (int)f_len;
             char *_buf = decompress(zip_type, buf, f_len, file_full_path, &_buf_len);
             if (_buf == NULL || _buf_len == 0) {
                 log_err("Cannot decompress zipped file %s", file_full_path);
                 goto cleanup;
             }
-            search_buf(_buf, _buf_len, file_full_path);
+            matches_count = search_buf(_buf, _buf_len, file_full_path);
             free(_buf);
+#endif
             goto cleanup;
         }
     }
 
-    search_buf(buf, f_len, file_full_path);
+    matches_count = search_buf(buf, f_len, file_full_path);
 
 cleanup:
 
+    if (opts.print_nonmatching_files && matches_count == 0) {
+        pthread_mutex_lock(&print_mtx);
+        print_path(file_full_path, opts.path_sep);
+        pthread_mutex_unlock(&print_mtx);
+        opts.match_found = 1;
+    }
+
+    print_cleanup_context();
     if (buf != NULL) {
 #ifdef _WIN32
         UnmapViewOfFile(buf);
 #else
-        munmap(buf, f_len);
+        if (opts.mmap) {
+            if (buf != MAP_FAILED) {
+                munmap(buf, f_len);
+            }
+        } else {
+            free(buf);
+        }
 #endif
     }
     if (fd != -1) {
@@ -421,6 +533,8 @@ void search_dir(ignores *ig, const char *base_path, const char *path, const int
     struct dirent *dir = NULL;
     scandir_baton_t scandir_baton;
     int results = 0;
+    size_t base_path_len = 0;
+    const char *path_start = path;
 
     char *dir_full_path = NULL;
     const char *ignore_file = NULL;
@@ -435,26 +549,29 @@ void search_dir(ignores *ig, const char *base_path, const char *path, const int
         return;
     }
 
-    /* find agignore/gitignore/hgignore/etc files to load ignore patterns from */
+    /* find .*ignore files to load ignore patterns from */
     for (i = 0; opts.skip_vcs_ignores ? (i == 0) : (ignore_pattern_files[i] != NULL); i++) {
         ignore_file = ignore_pattern_files[i];
         ag_asprintf(&dir_full_path, "%s/%s", path, ignore_file);
-        if (strcmp(SVN_DIR, ignore_file) == 0) {
-            load_svn_ignore_patterns(ig, dir_full_path);
-        } else {
-            load_ignore_patterns(ig, dir_full_path);
-        }
+        load_ignore_patterns(ig, dir_full_path);
         free(dir_full_path);
         dir_full_path = NULL;
     }
 
-    if (opts.path_to_agignore) {
-        load_ignore_patterns(ig, opts.path_to_agignore);
+    /* path_start is the part of path that isn't in base_path
+     * base_path will have a trailing '/' because we put it there in parse_options
+     */
+    base_path_len = base_path ? strlen(base_path) : 0;
+    for (i = 0; ((size_t)i < base_path_len) && (path[i]) && (base_path[i] == path[i]); i++) {
+        path_start = path + i + 1;
     }
+    log_debug("search_dir: path is '%s', base_path is '%s', path_start is '%s'", path, base_path, path_start);
 
     scandir_baton.ig = ig;
     scandir_baton.base_path = base_path;
-    scandir_baton.base_path_len = base_path ? strlen(base_path) : 0;
+    scandir_baton.base_path_len = base_path_len;
+    scandir_baton.path_start = path_start;
+
     results = ag_scandir(path, &dir_list, &filename_filter, &scandir_baton);
     if (results == 0) {
         log_debug("No results found in directory %s", path);

src/search.h

@@ -23,15 +23,17 @@
 #include <pthread.h>
 #endif
 
+#include "decompress.h"
 #include "ignore.h"
 #include "log.h"
 #include "options.h"
 #include "print.h"
-#include "util.h"
 #include "uthash.h"
+#include "util.h"
 
-size_t alpha_skip_lookup[256];
-size_t *find_skip_lookup;
+extern size_t alpha_skip_lookup[256];
+extern size_t *find_skip_lookup;
+extern uint8_t h_table[H_SIZE] __attribute__((aligned(64)));
 
 struct work_queue_t {
     char *path;
@@ -39,13 +41,12 @@ struct work_queue_t {
 };
 typedef struct work_queue_t work_queue_t;
 
-work_queue_t *work_queue;
-work_queue_t *work_queue_tail;
-int done_adding_files;
-pthread_cond_t files_ready;
-pthread_mutex_t print_mtx;
-pthread_mutex_t stats_mtx;
-pthread_mutex_t work_queue_mtx;
+extern work_queue_t *work_queue;
+extern work_queue_t *work_queue_tail;
+extern int done_adding_files;
+extern pthread_cond_t files_ready;
+extern pthread_mutex_t stats_mtx;
+extern pthread_mutex_t work_queue_mtx;
 
 
 /* For symlink loop detection */
@@ -63,11 +64,11 @@ typedef struct {
     UT_hash_handle hh;
 } symdir_t;
 
-symdir_t *symhash;
+extern symdir_t *symhash;
 
-void search_buf(const char *buf, const size_t buf_len,
-                const char *dir_full_path);
-void search_stream(FILE *stream, const char *path);
+ssize_t search_buf(const char *buf, const size_t buf_len,
+                   const char *dir_full_path);
+ssize_t search_stream(FILE *stream, const char *path);
 void search_file(const char *file_full_path);
 
 void *search_file_worker(void *i);

src/uthash.h

@@ -24,9 +24,9 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifndef UTHASH_H
 #define UTHASH_H
 
-#include <string.h> /* memcmp,strlen */
 #include <stddef.h> /* ptrdiff_t */
 #include <stdlib.h> /* exit() */
+#include <string.h> /* memcmp,strlen */
 
 /* These macros use decltype or the earlier __typeof GNU extension.
    As decltype is only available in newer compilers (VS2010 or gcc 4.3+
@@ -457,24 +457,34 @@ typedef unsigned char uint8_t;
         switch (_hj_k) {                                                                                                         \
             case 11:                                                                                                             \
                 hashv += ((unsigned)_hj_key[10] << 24);                                                                          \
+            /* fall through */                                                                                                   \
             case 10:                                                                                                             \
                 hashv += ((unsigned)_hj_key[9] << 16);                                                                           \
+            /* fall through */                                                                                                   \
             case 9:                                                                                                              \
                 hashv += ((unsigned)_hj_key[8] << 8);                                                                            \
+            /* fall through */                                                                                                   \
             case 8:                                                                                                              \
                 _hj_j += ((unsigned)_hj_key[7] << 24);                                                                           \
+            /* fall through */                                                                                                   \
             case 7:                                                                                                              \
                 _hj_j += ((unsigned)_hj_key[6] << 16);                                                                           \
+            /* fall through */                                                                                                   \
             case 6:                                                                                                              \
                 _hj_j += ((unsigned)_hj_key[5] << 8);                                                                            \
+            /* fall through */                                                                                                   \
             case 5:                                                                                                              \
                 _hj_j += _hj_key[4];                                                                                             \
+            /* fall through */                                                                                                   \
             case 4:                                                                                                              \
                 _hj_i += ((unsigned)_hj_key[3] << 24);                                                                           \
+            /* fall through */                                                                                                   \
             case 3:                                                                                                              \
                 _hj_i += ((unsigned)_hj_key[2] << 16);                                                                           \
+            /* fall through */                                                                                                   \
             case 2:                                                                                                              \
                 _hj_i += ((unsigned)_hj_key[1] << 8);                                                                            \
+            /* fall through */                                                                                                   \
             case 1:                                                                                                              \
                 _hj_i += _hj_key[0];                                                                                             \
         }                                                                                                                        \

src/util.c

@@ -1,14 +1,15 @@
 #include <ctype.h>
-#include <string.h>
-#include <stdio.h>
 #include <stdarg.h>
+#include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 #include <sys/stat.h>
 
-#include "util.h"
 #include "config.h"
+#include "util.h"
 
 #ifdef _WIN32
+#include <windows.h>
 #define flockfile(x)
 #define funlockfile(x)
 #define getc_unlocked(x) getc(x)
@@ -20,6 +21,8 @@
     }                                     \
     return ptr;
 
+FILE *out_fd = NULL;
+ag_stats stats;
 void *ag_malloc(size_t size) {
     void *ptr = malloc(size);
     CHECK_AND_RETURN(ptr)
@@ -147,14 +150,47 @@ size_t ag_max(size_t a, size_t b) {
     return a;
 }
 
+size_t ag_min(size_t a, size_t b) {
+    if (b < a) {
+        return b;
+    }
+    return a;
+}
+
+void generate_hash(const char *find, const size_t f_len, uint8_t *h_table, const int case_sensitive) {
+    int i;
+    for (i = f_len - sizeof(uint16_t); i >= 0; i--) {
+        // Add all 2^sizeof(uint16_t) combinations of capital letters to the hash table
+        int caps_set;
+        for (caps_set = 0; caps_set < (1 << sizeof(uint16_t)); caps_set++) {
+            word_t word;
+            memcpy(&word.as_chars, find + i, sizeof(uint16_t));
+            int cap_index;
+            // Capitalize the letters whose corresponding bits in caps_set are 1
+            for (cap_index = 0; caps_set >> cap_index; cap_index++) {
+                if ((caps_set >> cap_index) & 1)
+                    word.as_chars[cap_index] -= 'a' - 'A';
+            }
+            size_t h;
+            // Find next free cell
+            for (h = word.as_word % H_SIZE; h_table[h]; h = (h + 1) % H_SIZE)
+                ;
+            h_table[h] = i + 1;
+            // Don't add capital letters if case sensitive
+            if (case_sensitive)
+                break;
+        }
+    }
+}
+
 /* Boyer-Moore strstr */
 const char *boyer_moore_strnstr(const char *s, const char *find, const size_t s_len, const size_t f_len,
-                                const size_t alpha_skip_lookup[], const size_t *find_skip_lookup) {
+                                const size_t alpha_skip_lookup[], const size_t *find_skip_lookup, const int case_insensitive) {
     ssize_t i;
     size_t pos = f_len - 1;
 
     while (pos < s_len) {
-        for (i = f_len - 1; i >= 0 && s[pos] == find[i]; pos--, i--) {
+        for (i = f_len - 1; i >= 0 && (case_insensitive ? tolower(s[pos]) : s[pos]) == find[i]; pos--, i--) {
         }
         if (i < 0) {
             return s + pos + 1;
@@ -165,34 +201,44 @@ const char *boyer_moore_strnstr(const char *s, const char *find, const size_t s_
     return NULL;
 }
 
-/* Copy-pasted from above. Yes I know this is bad. One day I might even fix it. */
-const char *boyer_moore_strncasestr(const char *s, const char *find, const size_t s_len, const size_t f_len,
-                                    const size_t alpha_skip_lookup[], const size_t *find_skip_lookup) {
-    ssize_t i;
-    size_t pos = f_len - 1;
+// Clang's -fsanitize=alignment (included in -fsanitize=undefined) will flag
+// the intentional unaligned access here, so suppress it for this function
+NO_SANITIZE_ALIGNMENT const char *hash_strnstr(const char *s, const char *find, const size_t s_len, const size_t f_len, uint8_t *h_table, const int case_sensitive) {
+    if (s_len < f_len)
+        return NULL;
 
-    while (pos < s_len) {
-        for (i = f_len - 1; i >= 0 && tolower(s[pos]) == find[i]; pos--, i--) {
+    // Step through s
+    const size_t step = f_len - sizeof(uint16_t) + 1;
+    size_t s_i = f_len - sizeof(uint16_t);
+    for (; s_i <= s_len - f_len; s_i += step) {
+        size_t h;
+        for (h = *(const uint16_t *)(s + s_i) % H_SIZE; h_table[h]; h = (h + 1) % H_SIZE) {
+            const char *R = s + s_i - (h_table[h] - 1);
+            size_t i;
+            // Check putative match
+            for (i = 0; i < f_len; i++) {
+                if ((case_sensitive ? R[i] : tolower(R[i])) != find[i])
+                    goto next_hash_cell;
+            }
+            return R; // Found
+        next_hash_cell:;
         }
-        if (i < 0) {
-            return s + pos + 1;
-        }
-        pos += ag_max(alpha_skip_lookup[(unsigned char)s[pos]], find_skip_lookup[i]);
     }
-
+    // Check tail
+    for (s_i = s_i - step + 1; s_i <= s_len - f_len; s_i++) {
+        size_t i;
+        const char *R = s + s_i;
+        for (i = 0; i < f_len; i++) {
+            char s_c = case_sensitive ? R[i] : tolower(R[i]);
+            if (s_c != find[i])
+                goto next_start;
+        }
+        return R;
+    next_start:;
+    }
     return NULL;
 }
 
-strncmp_fp get_strstr(enum case_behavior casing) {
-    strncmp_fp ag_strncmp_fp = &boyer_moore_strnstr;
-
-    if (casing == CASE_INSENSITIVE) {
-        ag_strncmp_fp = &boyer_moore_strncasestr;
-    }
-
-    return ag_strncmp_fp;
-}
-
 size_t invert_matches(const char *buf, const size_t buf_len, match_t matches[], size_t matches_len) {
     size_t i;
     size_t match_read_index = 0;
@@ -291,6 +337,7 @@ int is_binary(const void *buf, const size_t buf_len) {
     size_t i;
 
     if (buf_len == 0) {
+        /* Is an empty file binary? Is it text? */
         return 0;
     }
 
@@ -299,7 +346,7 @@ int is_binary(const void *buf, const size_t buf_len) {
         return 0;
     }
 
-    if (buf_len >= 4 && strncmp(buf, "%PDF-", 5) == 0) {
+    if (buf_len >= 5 && strncmp(buf, "%PDF-", 5) == 0) {
         /* PDF. This is binary. */
         return 1;
     }
@@ -379,7 +426,7 @@ int binary_search(const char *needle, char **haystack, int start, int end) {
         return -1;
     }
 
-    mid = (start + end) / 2; /* can screw up on arrays with > 2 billion elements */
+    mid = start + ((end - start) / 2);
 
     rc = strcmp(needle, haystack[mid]);
     if (rc < 0) {
@@ -435,13 +482,20 @@ int is_directory(const char *path, const struct dirent *d) {
         free(full_path);
         return FALSE;
     }
+#ifdef _WIN32
+    int is_dir = GetFileAttributesA(full_path) & FILE_ATTRIBUTE_DIRECTORY;
+#else
+    int is_dir = S_ISDIR(s.st_mode);
+#endif
     free(full_path);
-    return S_ISDIR(s.st_mode);
+    return is_dir;
 }
 
 int is_symlink(const char *path, const struct dirent *d) {
 #ifdef _WIN32
-    return 0;
+    char full_path[MAX_PATH + 1] = { 0 };
+    sprintf(full_path, "%s\\%s", path, d->d_name);
+    return (GetFileAttributesA(full_path) & FILE_ATTRIBUTE_REPARSE_POINT);
 #else
 #ifdef HAVE_DIRENT_DTYPE
     /* Some filesystems, e.g. ReiserFS, always return a type DT_UNKNOWN from readdir or scandir. */
@@ -464,8 +518,8 @@ int is_symlink(const char *path, const struct dirent *d) {
 
 int is_named_pipe(const char *path, const struct dirent *d) {
 #ifdef HAVE_DIRENT_DTYPE
-    if (d->d_type != DT_UNKNOWN) {
-        return d->d_type == DT_FIFO;
+    if (d->d_type != DT_UNKNOWN && d->d_type != DT_LNK) {
+        return d->d_type == DT_FIFO || d->d_type == DT_SOCK;
     }
 #endif
     char *full_path;
@@ -476,7 +530,11 @@ int is_named_pipe(const char *path, const struct dirent *d) {
         return FALSE;
     }
     free(full_path);
-    return S_ISFIFO(s.st_mode);
+    return S_ISFIFO(s.st_mode)
+#ifdef S_ISSOCK
+           || S_ISSOCK(s.st_mode)
+#endif
+        ;
 }
 
 void ag_asprintf(char **ret, const char *fmt, ...) {
@@ -565,10 +623,10 @@ ssize_t getline(char **lineptr, size_t *n, FILE *stream) {
 }
 #endif
 
-ssize_t buf_getline(char **line, char *buf, size_t buf_len, size_t buf_offset) {
-    char *cur = buf + buf_offset;
+ssize_t buf_getline(const char **line, const char *buf, const size_t buf_len, const size_t buf_offset) {
+    const char *cur = buf + buf_offset;
     ssize_t i;
-    for (i = 0; cur[i] != '\n' && (buf_offset + i < buf_len); i++) {
+    for (i = 0; (buf_offset + i < buf_len) && cur[i] != '\n'; i++) {
     }
     *line = cur;
     return i;

src/util.h

@@ -3,16 +3,16 @@
 
 #include <dirent.h>
 #include <pcre.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <string.h>
-#include <stdio.h>
 #include <sys/time.h>
 
 #include "config.h"
 #include "log.h"
 #include "options.h"
 
-FILE *out_fd;
+extern FILE *out_fd;
 
 #ifndef TRUE
 #define TRUE 1
@@ -22,6 +22,14 @@ FILE *out_fd;
 #define FALSE 0
 #endif
 
+#define H_SIZE (64 * 1024)
+
+#ifdef __clang__
+#define NO_SANITIZE_ALIGNMENT __attribute__((no_sanitize("alignment")))
+#else
+#define NO_SANITIZE_ALIGNMENT
+#endif
+
 void *ag_malloc(size_t size);
 void *ag_realloc(void *ptr, size_t size);
 void *ag_calloc(size_t nelem, size_t elsize);
@@ -34,24 +42,22 @@ typedef struct {
 } match_t;
 
 typedef struct {
-    long total_bytes;
-    long total_files;
-    long total_matches;
-    long total_file_matches;
+    size_t total_bytes;
+    size_t total_files;
+    size_t total_matches;
+    size_t total_file_matches;
     struct timeval time_start;
     struct timeval time_end;
 } ag_stats;
 
-typedef enum {
-    AG_NO_COMPRESSION,
-    AG_GZIP,
-    AG_COMPRESS,
-    AG_ZIP
-} ag_compression_type;
 
-ag_stats stats;
+extern ag_stats stats;
 
-typedef const char *(*strncmp_fp)(const char *, const char *, const size_t, const size_t, const size_t[], const size_t *);
+/* Union to translate between chars and words without violating strict aliasing */
+typedef union {
+    char as_chars[sizeof(uint16_t)];
+    uint16_t as_word;
+} word_t;
 
 void free_strings(char **strs, const size_t strs_len);
 
@@ -59,23 +65,20 @@ void generate_alpha_skip(const char *find, size_t f_len, size_t skip_lookup[], c
 int is_prefix(const char *s, const size_t s_len, const size_t pos, const int case_sensitive);
 size_t suffix_len(const char *s, const size_t s_len, const size_t pos, const int case_sensitive);
 void generate_find_skip(const char *find, const size_t f_len, size_t **skip_lookup, const int case_sensitive);
+void generate_hash(const char *find, const size_t f_len, uint8_t *H, const int case_sensitive);
 
 /* max is already defined on spec-violating compilers such as MinGW */
 size_t ag_max(size_t a, size_t b);
+size_t ag_min(size_t a, size_t b);
 
 const char *boyer_moore_strnstr(const char *s, const char *find, const size_t s_len, const size_t f_len,
-                                const size_t alpha_skip_lookup[], const size_t *find_skip_lookup);
-const char *boyer_moore_strncasestr(const char *s, const char *find, const size_t s_len, const size_t f_len,
-                                    const size_t alpha_skip_lookup[], const size_t *find_skip_lookup);
-
-strncmp_fp get_strstr(enum case_behavior opts);
+                                const size_t alpha_skip_lookup[], const size_t *find_skip_lookup, const int case_insensitive);
+const char *hash_strnstr(const char *s, const char *find, const size_t s_len, const size_t f_len, uint8_t *h_table, const int case_sensitive);
 
 size_t invert_matches(const char *buf, const size_t buf_len, match_t matches[], size_t matches_len);
 void realloc_matches(match_t **matches, size_t *matches_size, size_t matches_len);
 void compile_study(pcre **re, pcre_extra **re_extra, char *q, const int pcre_opts, const int study_opts);
 
-void *decompress(const ag_compression_type zip_type, const void *buf, const int buf_len, const char *dir_full_path, int *new_buf_len);
-ag_compression_type is_zipped(const void *buf, const int buf_len);
 
 int is_binary(const void *buf, const size_t buf_len);
 int is_regex(const char *query);
@@ -95,7 +98,7 @@ void die(const char *fmt, ...);
 
 void ag_asprintf(char **ret, const char *fmt, ...);
 
-ssize_t buf_getline(char **line, char *buf, size_t buf_len, size_t buf_offset);
+ssize_t buf_getline(const char **line, const char *buf, const size_t buf_len, const size_t buf_offset);
 
 #ifndef HAVE_FGETLN
 char *fgetln(FILE *fp, size_t *lenp);

src/zfile.c

@@ -0,0 +1,403 @@
+#ifdef __FreeBSD__
+#include <sys/endian.h>
+#endif
+#include <sys/types.h>
+
+#ifdef __CYGWIN__
+typedef _off64_t off64_t;
+#endif
+
+#include <assert.h>
+#include <errno.h>
+#include <inttypes.h>
+#include <limits.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "config.h"
+
+#ifdef HAVE_ERR_H
+#include <err.h>
+#endif
+#ifdef HAVE_ZLIB_H
+#include <zlib.h>
+#endif
+#ifdef HAVE_LZMA_H
+#include <lzma.h>
+#endif
+
+#include "decompress.h"
+
+#if HAVE_FOPENCOOKIE
+
+#define min(a, b) ({				\
+	__typeof (a) _a = (a);			\
+	__typeof (b) _b = (b);			\
+	_a < _b ? _a : _b; })
+
+static cookie_read_function_t zfile_read;
+static cookie_seek_function_t zfile_seek;
+static cookie_close_function_t zfile_close;
+
+static const cookie_io_functions_t zfile_io = {
+    .read = zfile_read,
+    .write = NULL,
+    .seek = zfile_seek,
+    .close = zfile_close,
+};
+
+#define KB (1024)
+struct zfile {
+    FILE *in;              // Source FILE stream
+    uint64_t logic_offset, // Logical offset in output (forward seeks)
+        decode_offset,     // Where we've decoded to
+        actual_len;
+    uint32_t outbuf_start;
+
+    ag_compression_type ctype;
+
+    union {
+        z_stream gz;
+        lzma_stream lzma;
+    } stream;
+
+    uint8_t inbuf[32 * KB];
+    uint8_t outbuf[256 * KB];
+    bool eof;
+};
+
+#define CAVAIL_IN(c) ((c)->ctype == AG_GZIP ? (c)->stream.gz.avail_in : (c)->stream.lzma.avail_in)
+#define CNEXT_OUT(c) ((c)->ctype == AG_GZIP ? (c)->stream.gz.next_out : (c)->stream.lzma.next_out)
+
+static int
+zfile_cookie_init(struct zfile *cookie) {
+#ifdef HAVE_LZMA_H
+    lzma_ret lzrc;
+#endif
+    int rc;
+
+    assert(cookie->logic_offset == 0);
+    assert(cookie->decode_offset == 0);
+
+    cookie->actual_len = 0;
+
+    switch (cookie->ctype) {
+#ifdef HAVE_ZLIB_H
+        case AG_GZIP:
+            memset(&cookie->stream.gz, 0, sizeof cookie->stream.gz);
+            rc = inflateInit2(&cookie->stream.gz, 32 + 15);
+            if (rc != Z_OK) {
+                log_err("Unable to initialize zlib: %s", zError(rc));
+                return EIO;
+            }
+            cookie->stream.gz.next_in = NULL;
+            cookie->stream.gz.avail_in = 0;
+            cookie->stream.gz.next_out = cookie->outbuf;
+            cookie->stream.gz.avail_out = sizeof cookie->outbuf;
+            break;
+#endif
+#ifdef HAVE_LZMA_H
+        case AG_XZ:
+            cookie->stream.lzma = (lzma_stream)LZMA_STREAM_INIT;
+            lzrc = lzma_auto_decoder(&cookie->stream.lzma, -1, 0);
+            if (lzrc != LZMA_OK) {
+                log_err("Unable to initialize lzma_auto_decoder: %d", lzrc);
+                return EIO;
+            }
+            cookie->stream.lzma.next_in = NULL;
+            cookie->stream.lzma.avail_in = 0;
+            cookie->stream.lzma.next_out = cookie->outbuf;
+            cookie->stream.lzma.avail_out = sizeof cookie->outbuf;
+            break;
+#endif
+        default:
+            log_err("Unsupported compression type: %d", cookie->ctype);
+            return EINVAL;
+    }
+
+
+    cookie->outbuf_start = 0;
+    cookie->eof = false;
+    return 0;
+}
+
+static void
+zfile_cookie_cleanup(struct zfile *cookie) {
+    switch (cookie->ctype) {
+#ifdef HAVE_ZLIB_H
+        case AG_GZIP:
+            inflateEnd(&cookie->stream.gz);
+            break;
+#endif
+#ifdef HAVE_LZMA_H
+        case AG_XZ:
+            lzma_end(&cookie->stream.lzma);
+            break;
+#endif
+        default:
+            /* Compiler false positive - unreachable. */
+            break;
+    }
+}
+
+/*
+ * Open compressed file 'path' as a (forward-)seekable (and rewindable),
+ * read-only stream.
+ */
+FILE *
+decompress_open(int fd, const char *mode, ag_compression_type ctype) {
+    struct zfile *cookie;
+    FILE *res, *in;
+    int error;
+
+    cookie = NULL;
+    in = res = NULL;
+    if (strstr(mode, "w") || strstr(mode, "a")) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    in = fdopen(fd, mode);
+    if (in == NULL)
+        goto out;
+
+    /*
+	 * No validation of compression type is done -- file is assumed to
+	 * match input.  In Ag, the compression type is already detected, so
+	 * that's ok.
+	 */
+    cookie = malloc(sizeof *cookie);
+    if (cookie == NULL) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    cookie->in = in;
+    cookie->logic_offset = 0;
+    cookie->decode_offset = 0;
+    cookie->ctype = ctype;
+
+    error = zfile_cookie_init(cookie);
+    if (error != 0) {
+        errno = error;
+        goto out;
+    }
+
+    res = fopencookie(cookie, mode, zfile_io);
+
+out:
+    if (res == NULL) {
+        if (in != NULL)
+            fclose(in);
+        if (cookie != NULL)
+            free(cookie);
+    }
+    return res;
+}
+
+/*
+ * Return number of bytes into buf, 0 on EOF, -1 on error.  Update stream
+ * offset.
+ */
+static ssize_t
+zfile_read(void *cookie_, char *buf, size_t size) {
+    struct zfile *cookie = cookie_;
+    size_t nb, ignorebytes;
+    ssize_t total = 0;
+    lzma_ret lzret;
+    int ret;
+
+    assert(size <= SSIZE_MAX);
+
+    if (size == 0)
+        return 0;
+
+    if (cookie->eof)
+        return 0;
+
+    ret = Z_OK;
+    lzret = LZMA_OK;
+
+    ignorebytes = cookie->logic_offset - cookie->decode_offset;
+    assert(ignorebytes == 0);
+
+    do {
+        size_t inflated;
+
+        /* Drain output buffer first */
+        while (CNEXT_OUT(cookie) >
+               &cookie->outbuf[cookie->outbuf_start]) {
+            size_t left = CNEXT_OUT(cookie) -
+                          &cookie->outbuf[cookie->outbuf_start];
+            size_t ignoreskip = min(ignorebytes, left);
+            size_t toread;
+
+            if (ignoreskip > 0) {
+                ignorebytes -= ignoreskip;
+                left -= ignoreskip;
+                cookie->outbuf_start += ignoreskip;
+                cookie->decode_offset += ignoreskip;
+            }
+
+            // Ran out of output before we seek()ed up.
+            if (ignorebytes > 0)
+                break;
+
+            toread = min(left, size);
+            memcpy(buf, &cookie->outbuf[cookie->outbuf_start],
+                   toread);
+
+            buf += toread;
+            size -= toread;
+            left -= toread;
+            cookie->outbuf_start += toread;
+            cookie->decode_offset += toread;
+            cookie->logic_offset += toread;
+            total += toread;
+
+            if (size == 0)
+                break;
+        }
+
+        if (size == 0)
+            break;
+
+        /*
+		 * If we have not satisfied read, the output buffer must be
+		 * empty.
+		 */
+        assert(cookie->stream.gz.next_out ==
+               &cookie->outbuf[cookie->outbuf_start]);
+
+        if ((cookie->ctype == AG_XZ && lzret == LZMA_STREAM_END) ||
+            (cookie->ctype == AG_GZIP && ret == Z_STREAM_END)) {
+            cookie->eof = true;
+            break;
+        }
+
+        /* Read more input if empty */
+        if (CAVAIL_IN(cookie) == 0) {
+            nb = fread(cookie->inbuf, 1, sizeof cookie->inbuf,
+                       cookie->in);
+            if (ferror(cookie->in)) {
+                warn("error read core");
+                exit(1);
+            }
+            if (nb == 0 && feof(cookie->in)) {
+                warn("truncated file");
+                exit(1);
+            }
+            if (cookie->ctype == AG_XZ) {
+                cookie->stream.lzma.avail_in = nb;
+                cookie->stream.lzma.next_in = cookie->inbuf;
+            } else {
+                cookie->stream.gz.avail_in = nb;
+                cookie->stream.gz.next_in = cookie->inbuf;
+            }
+        }
+
+        /* Reset stream state to beginning of output buffer */
+        if (cookie->ctype == AG_XZ) {
+            cookie->stream.lzma.next_out = cookie->outbuf;
+            cookie->stream.lzma.avail_out = sizeof cookie->outbuf;
+        } else {
+            cookie->stream.gz.next_out = cookie->outbuf;
+            cookie->stream.gz.avail_out = sizeof cookie->outbuf;
+        }
+        cookie->outbuf_start = 0;
+
+        if (cookie->ctype == AG_GZIP) {
+            ret = inflate(&cookie->stream.gz, Z_NO_FLUSH);
+            if (ret != Z_OK && ret != Z_STREAM_END) {
+                log_err("Found mem/data error while decompressing zlib stream: %s", zError(ret));
+                return -1;
+            }
+        } else {
+            lzret = lzma_code(&cookie->stream.lzma, LZMA_RUN);
+            if (lzret != LZMA_OK && lzret != LZMA_STREAM_END) {
+                log_err("Found mem/data error while decompressing xz/lzma stream: %d", lzret);
+                return -1;
+            }
+        }
+        inflated = CNEXT_OUT(cookie) - &cookie->outbuf[0];
+        cookie->actual_len += inflated;
+    } while (!ferror(cookie->in) && size > 0);
+
+    assert(total <= SSIZE_MAX);
+    return total;
+}
+
+static int
+zfile_seek(void *cookie_, off64_t *offset_, int whence) {
+    struct zfile *cookie = cookie_;
+    off64_t new_offset = 0, offset = *offset_;
+
+    if (whence == SEEK_SET) {
+        new_offset = offset;
+    } else if (whence == SEEK_CUR) {
+        new_offset = (off64_t)cookie->logic_offset + offset;
+    } else {
+        /* SEEK_END not ok */
+        return -1;
+    }
+
+    if (new_offset < 0)
+        return -1;
+
+    /* Backward seeks to anywhere but 0 are not ok */
+    if (new_offset < (off64_t)cookie->logic_offset && new_offset != 0) {
+        return -1;
+    }
+
+    if (new_offset == 0) {
+        /* rewind(3) */
+        cookie->decode_offset = 0;
+        cookie->logic_offset = 0;
+        zfile_cookie_cleanup(cookie);
+        zfile_cookie_init(cookie);
+    } else if ((uint64_t)new_offset > cookie->logic_offset) {
+        /* Emulate forward seek by skipping ... */
+        char *buf;
+        const size_t bsz = 32 * 1024;
+
+        buf = malloc(bsz);
+        while ((uint64_t)new_offset > cookie->logic_offset) {
+            size_t diff = min(bsz,
+                              (uint64_t)new_offset - cookie->logic_offset);
+            ssize_t err = zfile_read(cookie_, buf, diff);
+            if (err < 0) {
+                free(buf);
+                return -1;
+            }
+
+            /* Seek past EOF gets positioned at EOF */
+            if (err == 0) {
+                assert(cookie->eof);
+                new_offset = cookie->logic_offset;
+                break;
+            }
+        }
+        free(buf);
+    }
+
+    assert(cookie->logic_offset == (uint64_t)new_offset);
+
+    *offset_ = new_offset;
+    return 0;
+}
+
+static int
+zfile_close(void *cookie_) {
+    struct zfile *cookie = cookie_;
+
+    zfile_cookie_cleanup(cookie);
+    fclose(cookie->in);
+    free(cookie);
+
+    return 0;
+}
+
+#endif /* HAVE_FOPENCOOKIE */

tests/adjacent_matches.t

@@ -1,8 +1,8 @@
 Setup:
 
   $ . $TESTDIR/setup.sh
-  $ alias ag="$TESTDIR/../ag --workers=1 --parallel --color"
-  $ echo blahfoofooblah > ./fooblah.txt
+  $ alias ag="$TESTDIR/../ag --noaffinity --workers=1 --parallel --color"
+  $ printf 'blahfoofooblah\n' > ./fooblah.txt
 
 Highlights are adjacent:
 

tests/case_sensitivity.t

@@ -1,8 +1,8 @@
 Setup:
 
   $ . $TESTDIR/setup.sh
-  $ echo Foo >> ./sample
-  $ echo bar >> ./sample
+  $ printf 'Foo\n' >> ./sample
+  $ printf 'bar\n' >> ./sample
 
 Smart case by default:
 
@@ -29,11 +29,17 @@ Case sensitive mode:
   1:Foo
   $ ag -s 'F.o' sample
   1:Foo
+
 Case insensitive mode:
 
-  $ ag foo -i sample
+  $ ag fOO -i sample
   1:Foo
-  $ ag foo --ignore-case sample
+  $ ag fOO --ignore-case sample
   1:Foo
   $ ag 'f.o' -i sample
   1:Foo
+
+Case insensitive file regex
+
+  $ ag -i  -g 'Samp.*'
+  sample

tests/color.t

@@ -2,9 +2,9 @@ Setup. Note that we have to turn --color on manually since ag detects that
 stdout isn't a tty when running in cram.
 
   $ . $TESTDIR/setup.sh
-  $ alias ag="$TESTDIR/../ag --workers=1 --parallel --color"
-  $ echo foo > ./blah.txt
-  $ echo bar >> ./blah.txt
+  $ alias ag="$TESTDIR/../ag --noaffinity --workers=1 --parallel --color"
+  $ printf 'foo\n' > ./blah.txt
+  $ printf 'bar\n' >> ./blah.txt
 
 Matches should contain colors:
 

tests/column.t

@@ -1,7 +1,7 @@
 Setup:
 
   $ . $TESTDIR/setup.sh
-  $ echo "blah\nblah2" > blah.txt
+  $ printf "blah\nblah2\n" > blah.txt
 
 Ensure column is correct:
 
@@ -12,7 +12,7 @@ Ensure column is correct:
 # Test ackmate output. Not quite right, but at least offsets are in the
 # ballpark instead of being 9 quintillion
 
-  $ ag --ackmate "blah\nb"
+  $ ag --ackmate "lah\nb"
   :blah.txt
   1;blah
-  2;0 6:blah2
+  2;1 5:blah2

tests/count.t

@@ -2,11 +2,11 @@ Setup:
 
   $ . $TESTDIR/setup.sh
   $ unalias ag
-  $ alias ag="$TESTDIR/../ag --nocolor --workers=1"
-  $ echo "blah" > blah.txt
-  $ echo "blah2" >> blah.txt
-  $ echo "blah_OTHER" > other_file.txt
-  $ echo "blah_OTHER" >> other_file.txt
+  $ alias ag="$TESTDIR/../ag --noaffinity --nocolor --workers=1"
+  $ printf "blah\n" > blah.txt
+  $ printf "blah2\n" >> blah.txt
+  $ printf "blah_OTHER\n" > other_file.txt
+  $ printf "blah_OTHER\n" >> other_file.txt
 
 Count matches:
 
@@ -16,7 +16,7 @@ Count matches:
 
 Count stream matches:
 
-  $ echo 'blah blah blah' | ag --count blah
+  $ printf 'blah blah blah\n' | ag --count blah
   3
 
 Count stream matches per line (not very useful since it does not print zero):

tests/ds_store_ignore.t

@@ -1,8 +1,8 @@
 Setup.
   $ . $TESTDIR/setup.sh
   $ mkdir -p dir0/dir1/dir2
-  $ echo '*.DS_Store' > dir0/.gitignore
-  $ echo blah > dir0/dir1/dir2/blah.txt
+  $ printf '*.DS_Store\n' > dir0/.ignore
+  $ printf 'blah\n' > dir0/dir1/dir2/blah.txt
   $ touch dir0/dir1/.DS_Store
 
 Find blah in blah.txt

tests/empty_environment.t

@@ -0,0 +1,9 @@
+Setup:
+
+  $ . $TESTDIR/setup.sh
+  $ printf "hello world\n" >test.txt
+
+Verify ag runs with an empty environment:
+
+  $ env -i $TESTDIR/../ag --noaffinity --nocolor --workers=1 --parallel hello
+  test.txt:1:hello world

tests/empty_match.t

@@ -1,7 +1,7 @@
 Setup.
   $ . $TESTDIR/setup.sh
   $ touch empty.txt
-  $ echo foo > nonempty.txt
+  $ printf 'foo\n' > nonempty.txt
 
 Zero-length match on an empty file should fail silently with return code 1
 
@@ -11,3 +11,12 @@ Zero-length match on an empty file should fail silently with return code 1
 A genuine zero-length match should succeed:
   $ ag "^" nonempty.txt
   1:foo
+
+Empty files should be listed with --unrestricted --files-with-matches (-ul)
+  $ ag -lu --stats | sed '$d' | sort # Remove the last line about timing which will differ
+  2 files contained matches
+  2 files searched
+  2 matches
+  4 bytes searched
+  empty.txt
+  nonempty.txt

tests/exitcodes.t

@@ -1,8 +1,8 @@
 Setup:
 
   $ . $TESTDIR/setup.sh
-  $ echo foo > ./exitcodes_test.txt
-  $ echo bar >> ./exitcodes_test.txt
+  $ printf 'foo\n' > ./exitcodes_test.txt
+  $ printf 'bar\n' >> ./exitcodes_test.txt
 
 Normal matching:
 

tests/fail/unicode_case_insensitive.t

@@ -0,0 +1,17 @@
+Setup:
+
+  $ . $TESTDIR/../setup.sh
+  $ printf "hello=你好\n" > test.txt
+  $ printf "hello=你好\n" >> test.txt
+
+Normal search:
+
+  $ $TESTDIR/../../ag --nocolor --workers=1 --parallel 你好
+  test.txt:1:hello=你好
+  test.txt:2:hello=你好
+
+Case-insensitive search:
+
+  $ $TESTDIR/../../ag --nocolor --workers=1 --parallel -i 你好
+  test.txt:1:hello=你好
+  test.txt:2:hello=你好

tests/fail/unicode_case_insensitive.t.err

@@ -0,0 +1,17 @@
+Setup:
+
+  $ . $TESTDIR/../setup.sh
+  $ printf "hello=你好\n" > test.txt
+  $ printf "hello=你好\n" >> test.txt
+
+Normal search:
+
+  $ $TESTDIR/../../ag --nocolor --workers=1 --parallel 你好
+  test.txt:1:hello=\xe4\xbd\xa0\xe5\xa5\xbd (esc)
+  test.txt:2:hello=\xe4\xbd\xa0\xe5\xa5\xbd (esc)
+
+Case-insensitive search:
+
+  $ $TESTDIR/../../ag --nocolor --workers=1 --parallel -i 你好
+  test.txt:1:hello=\xe4\xbd\xa0\xe5\xa5\xbd (esc)
+  test.txt:2:hello=\xe4\xbd\xa0\xe5\xa5\xbd (esc)

tests/files_with_matches.t

@@ -0,0 +1,63 @@
+Setup:
+
+  $ . $TESTDIR/setup.sh
+  $ printf 'foo\n' > ./foo.txt
+  $ printf 'bar\n' > ./bar.txt
+  $ printf 'foo\nbar\nbaz\n' > ./baz.txt
+  $ printf 'duck\nanother duck\nyet another duck\n' > ./duck.txt
+  $ cp duck.txt goose.txt
+  $ echo "GOOSE!!!" >> ./goose.txt
+
+Files with matches:
+
+  $ ag --files-with-matches foo foo.txt
+  foo.txt
+  $ ag --files-with-matches foo foo.txt bar.txt
+  foo.txt
+  $ ag --files-with-matches foo bar.txt
+  [1]
+  $ ag --files-with-matches foo foo.txt bar.txt baz.txt
+  foo.txt
+  baz.txt
+  $ ag --files-with-matches bar foo.txt bar.txt baz.txt
+  bar.txt
+  baz.txt
+  $ ag --files-with-matches foo bar.txt baz.txt
+  baz.txt
+
+Files without matches:
+(Prints names of files in which no line matches query)
+
+  $ ag --files-without-matches bar foo.txt
+  foo.txt
+  $ ag --files-without-matches bar foo.txt bar.txt
+  foo.txt
+  $ ag --files-without-matches bar bar.txt
+  [1]
+  $ ag --files-without-matches foo foo.txt bar.txt baz.txt
+  bar.txt
+  $ ag --files-without-matches bar foo.txt bar.txt baz.txt
+  foo.txt
+
+Files with inverted matches:
+(Prints names of files in which some line doesn't match query)
+
+  $ ag --files-with-matches --invert-match bar bar.txt
+  [1]
+  $ ag --files-with-matches --invert-match foo foo.txt bar.txt baz.txt
+  bar.txt
+  baz.txt
+  $ ag --files-with-matches --invert-match bar foo.txt bar.txt baz.txt
+  foo.txt
+  baz.txt
+
+Files without inverted matches:
+(Prints names of files in which no line doesn't match query,
+ i.e. where every line matches query)
+
+  $ ag --files-without-matches --invert-match duck duck.txt
+  duck.txt
+  $ ag --files-without-matches --invert-match duck goose.txt
+  [1]
+  $ ag --files-without-matches --invert-match duck duck.txt goose.txt
+  duck.txt

tests/filetype.t

@@ -0,0 +1,15 @@
+Setup:
+
+  $ . $TESTDIR/setup.sh
+  $ TEST_FILETYPE_EXT1=`ag --list-file-types | grep -E '^[ \t]+\..+' | head -n 1 | awk '{ print $1 }'`
+  $ TEST_FILETYPE_EXT2=`ag --list-file-types | grep -E '^[ \t]+\..+' | tail -n 1 | awk '{ print $1 }'`
+  $ TEST_FILETYPE_DIR=filetype_test
+  $ mkdir $TEST_FILETYPE_DIR
+  $ printf "This is filetype test1.\n" >  $TEST_FILETYPE_DIR/test.$TEST_FILETYPE_EXT1
+  $ printf "This is filetype test2.\n" >  $TEST_FILETYPE_DIR/test.$TEST_FILETYPE_EXT2
+
+Match only top file type:
+
+  $ TEST_FILETYPE_OPTION=`ag --list-file-types | grep -E '^[ \t]+--.+' | head -n 1 | awk '{ print $1 }'`
+  $ ag 'This is filetype test' --nofilename $TEST_FILETYPE_OPTION $TEST_FILETYPE_DIR
+  This is filetype test1.

tests/hidden_option.t

@@ -3,10 +3,10 @@ Setup:
   $ . $TESTDIR/setup.sh
   $ mkdir hidden_bug
   $ cd hidden_bug
-  $ echo "test" > a.txt
+  $ printf "test\n" > a.txt
   $ git init --quiet
   $ if [ ! -d .git/info ] ; then mkdir .git/info ; fi
-  $ echo "a.txt" > .git/info/exclude
+  $ printf "a.txt\n" > .git/info/exclude
 
   $ ag --ignore-dir .git test
   [1]
@@ -21,7 +21,7 @@ Setup:
   a.txt:1:test
 
   $ mkdir -p ./.hidden
-  $ echo 'whatever' > ./.hidden/a.txt
+  $ printf 'whatever\n' > ./.hidden/a.txt
 
   $ ag whatever
   [1]
@@ -29,7 +29,7 @@ Setup:
   $ ag --hidden whatever
   [1]
 
-  $ echo "" > .git/info/exclude
+  $ printf "\n" > .git/info/exclude
 
   $ ag whatever
   [1]

tests/ignore_abs_path.t

@@ -2,9 +2,9 @@ Setup:
 
   $ . $TESTDIR/setup.sh
   $ mkdir -p ./a/b/c
-  $ echo 'whatever1' > ./a/b/c/blah.yml
-  $ echo 'whatever2' > ./a/b/foo.yml
-  $ echo '/a/b/foo.yml' > ./.gitignore
+  $ printf 'whatever1\n' > ./a/b/c/blah.yml
+  $ printf 'whatever2\n' > ./a/b/foo.yml
+  $ printf '/a/b/foo.yml\n' > ./.ignore
 
 Ignore foo.yml but not blah.yml:
 

tests/ignore_absolute_search_path_with_glob.t

@@ -2,8 +2,8 @@ Setup:
 
   $ . $TESTDIR/setup.sh
   $ mkdir -p parent/multi-part
-  $ echo 'match1' > parent/multi-part/file1.txt
-  $ echo 'parent/multi-*' > .gitignore
+  $ printf 'match1\n' > parent/multi-part/file1.txt
+  $ printf 'parent/multi-*\n' > .ignore
 
 # Ignore directory specified by glob:
 

tests/ignore_backups.t

@@ -2,24 +2,24 @@ Setup:
 
   $ . $TESTDIR/setup.sh
   $ mkdir -p ./a/b/c
-  $ echo 'whatever1'  > ./a/b/c/foo.yml
-  $ echo 'whatever2'  > ./a/b/c/foo.yml~
-  $ echo 'whatever3'  > ./a/b/c/.foo.yml.swp
-  $ echo 'whatever4'  > ./a/b/c/.foo.yml.swo
-  $ echo 'whatever5'  > ./a/b/foo.yml
-  $ echo 'whatever6'  > ./a/b/foo.yml~
-  $ echo 'whatever7'  > ./a/b/.foo.yml.swp
-  $ echo 'whatever8'  > ./a/b/.foo.yml.swo
-  $ echo 'whatever9'  > ./a/foo.yml
-  $ echo 'whatever10' > ./a/foo.yml~
-  $ echo 'whatever11' > ./a/.foo.yml.swp
-  $ echo 'whatever12' > ./a/.foo.yml.swo
-  $ echo 'whatever13' > ./foo.yml
-  $ echo 'whatever14' > ./foo.yml~
-  $ echo 'whatever15' > ./.foo.yml.swp
-  $ echo 'whatever16' > ./.foo.yml.swo
-  $ echo '*~'         > ./.gitignore
-  $ echo '*.sw[po]'   >> ./.gitignore
+  $ printf 'whatever1\n'  > ./a/b/c/foo.yml
+  $ printf 'whatever2\n'  > ./a/b/c/foo.yml~
+  $ printf 'whatever3\n'  > ./a/b/c/.foo.yml.swp
+  $ printf 'whatever4\n'  > ./a/b/c/.foo.yml.swo
+  $ printf 'whatever5\n'  > ./a/b/foo.yml
+  $ printf 'whatever6\n'  > ./a/b/foo.yml~
+  $ printf 'whatever7\n'  > ./a/b/.foo.yml.swp
+  $ printf 'whatever8\n'  > ./a/b/.foo.yml.swo
+  $ printf 'whatever9\n'  > ./a/foo.yml
+  $ printf 'whatever10\n' > ./a/foo.yml~
+  $ printf 'whatever11\n' > ./a/.foo.yml.swp
+  $ printf 'whatever12\n' > ./a/.foo.yml.swo
+  $ printf 'whatever13\n' > ./foo.yml
+  $ printf 'whatever14\n' > ./foo.yml~
+  $ printf 'whatever15\n' > ./.foo.yml.swp
+  $ printf 'whatever16\n' > ./.foo.yml.swo
+  $ printf '*~\n'         > ./.ignore
+  $ printf '*.sw[po]\n'   >> ./.ignore
 
 Ignore all files except foo.yml
 

tests/ignore_examine_parent_ignorefiles.t

@@ -2,8 +2,8 @@ Setup:
 
   $ . $TESTDIR/setup.sh
   $ mkdir -p subdir
-  $ echo 'match1' > subdir/file1.txt
-  $ echo 'file1.txt' > .gitignore
+  $ printf 'match1\n' > subdir/file1.txt
+  $ printf 'file1.txt\n' > .ignore
 
 Ignore directory specified by name:
 

tests/ignore_extensions.t

@@ -1,15 +1,17 @@
 Setup:
 
   $ . $TESTDIR/setup.sh
-  $ echo '*.js' > .gitignore
-  $ echo '\n*.test.txt' >> .gitignore
-  $ echo 'targetA' > something.js
-  $ echo 'targetB' > aFile.test.txt
-  $ echo 'targetC' > aFile.txt
+  $ printf '*.js\n' > .ignore
+  $ printf '*.test.txt\n' >> .ignore
+  $ printf 'targetA\n' > something.js
+  $ printf 'targetB\n' > aFile.test.txt
+  $ printf 'targetC\n' > aFile.txt
+  $ printf 'targetG\n' > something.min.js
   $ mkdir -p subdir
-  $ echo 'targetD' > subdir/somethingElse.js
-  $ echo 'targetE' > subdir/anotherFile.test.txt
-  $ echo 'targetF' > subdir/anotherFile.txt
+  $ printf 'targetD\n' > subdir/somethingElse.js
+  $ printf 'targetE\n' > subdir/anotherFile.test.txt
+  $ printf 'targetF\n' > subdir/anotherFile.txt
+  $ printf 'targetH\n' > subdir/somethingElse.min.js
 
 Ignore patterns with single extension in root directory:
 
@@ -21,6 +23,11 @@ Ignore patterns with multiple extensions in root directory:
   $ ag "targetB"
   [1]
 
+*.js ignores *.min.js in root directory:
+
+  $ ag "targetG"
+  [1]
+
 Do not ignore patterns with partial extensions in root directory:
 
   $ ag "targetC"
@@ -36,6 +43,11 @@ Ignore patterns with multiple extensions in subdirectory:
   $ ag "targetE"
   [1]
 
+*.js ignores *.min.js in subdirectory:
+
+  $ ag "targetH"
+  [1]
+
 Do not ignore patterns with partial extensions in subdirectory:
 
   $ ag "targetF"

tests/ignore_gitignore.t

@@ -3,7 +3,7 @@ Setup:
   $ . $TESTDIR/setup.sh
   $ export HOME=$PWD
   $ printf '[core]\nexcludesfile = ~/.gitignore.global' >> $HOME/.gitconfig
-  $ echo 'PATTERN_MARKER' > .gitignore.global
+  $ printf 'PATTERN_MARKER\n' > .gitignore.global
 
 Test that the ignore pattern got picked up:
 

tests/ignore_invert.t

@@ -0,0 +1,12 @@
+Setup:
+
+  $ . $TESTDIR/setup.sh
+  $ printf 'blah1\n' > ./printme.txt
+  $ printf 'blah2\n' > ./dontprintme.c
+  $ printf '*\n' > ./.ignore
+  $ printf '!*.txt\n' >> ./.ignore
+
+Ignore .gitignore patterns but not .ignore patterns:
+
+  $ ag blah
+  printme.txt:1:blah1

tests/ignore_pattern_in_subdirectory.t

@@ -2,9 +2,9 @@ Setup:
 
   $ . $TESTDIR/setup.sh
   $ mkdir subdir
-  $ echo 'first' > file1.txt
-  $ echo 'second' > subdir/file2.txt
-  $ echo '*.txt' > .gitignore
+  $ printf 'first\n' > file1.txt
+  $ printf 'second\n' > subdir/file2.txt
+  $ printf '*.txt\n' > .gitignore
 
 Ignore file based on extension match:
 

tests/ignore_slash_in_subdir.t

@@ -0,0 +1,19 @@
+Setup:
+
+  $ . $TESTDIR/setup.sh
+  $ mkdir -p subdir/ignoredir
+  $ mkdir ignoredir
+  $ printf 'match1\n' > subdir/ignoredir/file1.txt
+  $ printf 'match1\n' > ignoredir/file1.txt
+  $ printf '/ignoredir\n' > subdir/.ignore
+
+Ignore file in subdir/ignoredir, but not in ignoredir:
+
+  $ ag match
+  ignoredir/file1.txt:1:match1
+
+From subdir, ignore file in subdir/ignoredir:
+
+  $ cd subdir
+  $ ag match
+  [1]

tests/ignore_subdir.t

@@ -2,11 +2,11 @@ Setup:
 
   $ . $TESTDIR/setup.sh
   $ mkdir -p ./a/b/c
-  $ echo 'whatever1' > ./a/b/c/blah.yml
-  $ echo 'whatever2' > ./a/b/foo.yml
-  $ echo 'a/b/foo.yml' > ./.gitignore
+  $ printf 'whatever1\n' > ./a/b/c/blah.yml
+  $ printf 'whatever2\n' > ./a/b/foo.yml
+  $ printf 'a/b/foo.yml\n' > ./.gitignore
 # TODO: have this work instead of the above
-# $ echo 'a/b/*.yml' > ./.gitignore
+# $ printf 'a/b/*.yml\n' > ./.gitignore
 
 Ignore foo.yml but not blah.yml:
 

tests/ignore_vcs.t

@@ -0,0 +1,19 @@
+Setup:
+
+  $ . $TESTDIR/setup.sh
+  $ printf 'whatever1\n' > ./always.txt
+  $ printf 'whatever2\n' > ./git.txt
+  $ printf 'whatever3\n' > ./text.txt
+  $ printf 'git.txt\n' > ./.gitignore
+  $ printf 'text.*\n' > ./.ignore
+
+Obey .gitignore and .ignore patterns:
+
+  $ ag whatever .
+  always.txt:1:whatever1
+
+Ignore .gitignore patterns but not .ignore patterns:
+
+  $ ag -U whatever . | sort
+  always.txt:1:whatever1
+  git.txt:1:whatever2

tests/invert_match.t

@@ -1,13 +1,13 @@
 Setup:
 
   $ . $TESTDIR/setup.sh
-  $ echo 'valid: 1' > ./blah.txt
-  $ echo 'some_string' >> ./blah.txt
-  $ echo 'valid: 654' >> ./blah.txt
-  $ echo 'some_other_string' >> ./blah.txt
-  $ echo 'valid: 0' >> ./blah.txt
-  $ echo 'valid: 23' >> ./blah.txt
-  $ echo 'valid: 0' >> ./blah.txt
+  $ printf 'valid: 1\n' > ./blah.txt
+  $ printf 'some_string\n' >> ./blah.txt
+  $ printf 'valid: 654\n' >> ./blah.txt
+  $ printf 'some_other_string\n' >> ./blah.txt
+  $ printf 'valid: 0\n' >> ./blah.txt
+  $ printf 'valid: 23\n' >> ./blah.txt
+  $ printf 'valid: 0\n' >> ./blah.txt
 
 Search for lines not matching "valid: 0" in blah.txt:
 

tests/line_width.t

@@ -0,0 +1,30 @@
+Setup:
+
+  $ . $TESTDIR/setup.sh
+  $ printf "12345678901234567890123456789012345678901234567890\n" >> ./blah.txt
+
+Truncate to width inside input line length:
+
+  $ ag -W 20 1 < ./blah.txt
+  blah.txt:1:12345678901234567890 [...]
+
+Truncate to width inside input line length, long-form:
+
+  $ ag --width 20 1 < ./blah.txt
+  blah.txt:1:12345678901234567890 [...]
+
+Truncate to width outside input line length:
+
+  $ ag -W 60 1 < ./blah.txt
+  blah.txt:1:12345678901234567890123456789012345678901234567890
+
+Truncate to width one less than input line length:
+
+  $ ag -W 49 1 < ./blah.txt
+  blah.txt:1:1234567890123456789012345678901234567890123456789 [...]
+
+Truncate to width exactly input line length:
+
+  $ ag -W 50 1 < ./blah.txt
+  blah.txt:1:12345678901234567890123456789012345678901234567890
+

tests/list_file_types.t

@@ -12,35 +12,80 @@ Language types are output:
     --ada
         .ada  .adb  .ads
   
+    --asciidoc
+        .adoc  .ad  .asc  .asciidoc
+  
+    --apl
+        .apl
+  
     --asm
         .asm  .s
   
+    --asp
+        .asp  .asa  .aspx  .asax  .ashx  .ascx  .asmx
+  
+    --aspx
+        .asp  .asa  .aspx  .asax  .ashx  .ascx  .asmx
+  
     --batch
         .bat  .cmd
   
+    --bazel
+        .bazel
+  
+    --bitbake
+        .bb  .bbappend  .bbclass  .inc
+  
     --cc
         .c  .h  .xs
   
     --cfmx
         .cfc  .cfm  .cfml
   
+    --chpl
+        .chpl
+  
     --clojure
-        .clj  .cljs  .cljc  .cljx
+        .clj  .cljs  .cljc  .cljx  .edn
   
     --coffee
         .coffee  .cjsx
   
+    --config
+        .config
+  
+    --coq
+        .coq  .g  .v
+  
     --cpp
-        .cpp  .cc  .C  .cxx  .m  .hpp  .hh  .h  .H  .hxx
+        .cpp  .cc  .C  .cxx  .m  .hpp  .hh  .h  .H  .hxx  .tpp
+  
+    --crystal
+        .cr  .ecr
   
     --csharp
         .cs
   
+    --cshtml
+        .cshtml
+  
     --css
         .css
   
+    --cython
+        .pyx  .pxd  .pxi
+  
     --delphi
-        .pas  .int  .dfm  .nfm  .dof  .dpk  .dproj  .groupproj  .bdsgroup  .bdsproj
+        .pas  .int  .dfm  .nfm  .dof  .dpk  .dpr  .dproj  .groupproj  .bdsgroup  .bdsproj
+  
+    --dlang
+        .d  .di
+  
+    --dot
+        .dot  .gv
+  
+    --dts
+        .dts  .dtsi
   
     --ebuild
         .ebuild  .eclass
@@ -49,13 +94,19 @@ Language types are output:
         .el
   
     --elixir
-        .ex  .exs
+        .ex  .eex  .exs
+  
+    --elm
+        .elm
   
     --erlang
         .erl  .hrl
   
+    --factor
+        .factor
+  
     --fortran
-        .f  .f77  .f90  .f95  .f03  .for  .ftn  .fpp
+        .f  .F  .f77  .f90  .F90  .f95  .f03  .for  .ftn  .fpp  .FPP
   
     --fsharp
         .fs  .fsi  .fsx
@@ -63,17 +114,29 @@ Language types are output:
     --gettext
         .po  .pot  .mo
   
+    --glsl
+        .vert  .tesc  .tese  .geom  .frag  .comp
+  
     --go
         .go
   
+    --gradle
+        .gradle
+  
     --groovy
-        .groovy  .gtmpl  .gpp  .grunit
+        .groovy  .gtmpl  .gpp  .grunit  .gradle
   
     --haml
         .haml
   
+    --handlebars
+        .hbs
+  
     --haskell
-        .hs  .lhs
+        .hs  .hsig  .lhs
+  
+    --haxe
+        .hx
   
     --hh
         .h
@@ -81,23 +144,44 @@ Language types are output:
     --html
         .htm  .html  .shtml  .xhtml
   
+    --idris
+        .idr  .ipkg  .lidr
+  
     --ini
         .ini
   
+    --ipython
+        .ipynb
+  
+    --isabelle
+        .thy
+  
+    --j
+        .ijs
+  
     --jade
         .jade
   
     --java
         .java  .properties
   
+    --jinja2
+        .j2
+  
     --js
-        .js  .jsx
+        .es6  .js  .jsx  .vue
   
     --json
         .json
   
     --jsp
-        .jsp  .jspx  .jhtm  .jhtml
+        .jsp  .jspx  .jhtm  .jhtml  .jspf  .tag  .tagf
+  
+    --julia
+        .jl
+  
+    --kotlin
+        .kt
   
     --less
         .less
@@ -108,6 +192,9 @@ Language types are output:
     --lisp
         .lisp  .lsp
   
+    --log
+        .log
+  
     --lua
         .lua
   
@@ -132,12 +219,21 @@ Language types are output:
     --mathematica
         .m  .wl
   
+    --md
+        .markdown  .mdown  .mdwn  .mkdn  .mkd  .md
+  
     --mercury
         .m  .moo
   
+    --naccess
+        .asa  .rsa
+  
     --nim
         .nim
   
+    --nix
+        .nix
+  
     --objc
         .m  .h
   
@@ -150,9 +246,15 @@ Language types are output:
     --octave
         .m
   
+    --org
+        .org
+  
     --parrot
         .pir  .pasm  .pmc  .ops  .pod  .pg  .tg
   
+    --pdb
+        .pdb
+  
     --perl
         .pl  .pm  .pm6  .pod  .t
   
@@ -162,8 +264,23 @@ Language types are output:
     --pike
         .pike  .pmod
   
+    --plist
+        .plist
+  
     --plone
-        .pt  .cpt  .metadata  .cpy  .py
+        .pt  .cpt  .metadata  .cpy  .py  .xml  .zcml
+  
+    --powershell
+        .ps1
+  
+    --proto
+        .proto
+  
+    --ps1
+        .ps1
+  
+    --pug
+        .pug
   
     --puppet
         .pp
@@ -171,17 +288,29 @@ Language types are output:
     --python
         .py
   
+    --qml
+        .qml
+  
     --racket
         .rkt  .ss  .scm
   
     --rake
-        .Rakefiles
+        .Rakefile
+  
+    --razor
+        .cshtml
+  
+    --restructuredtext
+        .rst
   
     --rs
         .rs
   
     --r
-        .R  .Rmd  .Rnw  .Rtex  .Rrst
+        .r  .R  .Rmd  .Rnw  .Rtex  .Rrst
+  
+    --rdoc
+        .rdoc
   
     --ruby
         .rb  .rhtml  .rjs  .rxml  .erb  .rake  .spec
@@ -202,7 +331,7 @@ Language types are output:
         .scm  .ss
   
     --shell
-        .sh  .bash  .csh  .tcsh  .ksh  .zsh
+        .sh  .bash  .csh  .tcsh  .ksh  .zsh  .fish
   
     --smalltalk
         .st
@@ -213,6 +342,9 @@ Language types are output:
     --sql
         .sql  .ctl
   
+    --stata
+        .do  .ado
+  
     --stylus
         .styl
   
@@ -222,12 +354,30 @@ Language types are output:
     --tcl
         .tcl  .itcl  .itk
   
+    --terraform
+        .tf  .tfvars
+  
     --tex
         .tex  .cls  .sty
   
+    --thrift
+        .thrift
+  
+    --tla
+        .tla
+  
     --tt
         .tt  .tt2  .ttml
   
+    --toml
+        .toml
+  
+    --ts
+        .ts  .tsx
+  
+    --twig
+        .twig
+  
     --vala
         .vala  .vapi
   
@@ -235,10 +385,10 @@ Language types are output:
         .bas  .cls  .frm  .ctl  .vb  .resx
   
     --velocity
-        .vm
+        .vm  .vtl  .vsl
   
     --verilog
-        .v  .vh  .sv
+        .v  .vh  .sv  .svh
   
     --vhdl
         .vhd  .vhdl
@@ -246,6 +396,12 @@ Language types are output:
     --vim
         .vim
   
+    --vue
+        .vue
+  
+    --wix
+        .wxi  .wxs
+  
     --wsdl
         .wsdl
   
@@ -253,8 +409,14 @@ Language types are output:
         .wadl
   
     --xml
-        .xml  .dtd  .xsl  .xslt  .ent
+        .xml  .dtd  .xsl  .xslt  .xsd  .ent  .tld  .plist  .wsdl
   
     --yaml
         .yaml  .yml
   
+    --zeek
+        .zeek  .bro  .bif
+  
+    --zephir
+        .zep
+  

tests/literal_word_regexp.t

@@ -0,0 +1,40 @@
+Setup:
+
+  $ . $TESTDIR/setup.sh
+  $ echo 'blah abc def' > blah1.txt
+  $ echo 'abc blah def' > blah2.txt
+  $ echo 'abc def blah' > blah3.txt
+  $ echo 'abcblah def' > blah4.txt
+  $ echo 'abc blahdef' >> blah4.txt
+  $ echo 'blahx blah' > blah5.txt
+  $ echo 'abcblah blah blah' > blah6.txt
+
+Match a word of the beginning:
+
+  $ ag -wF --column 'blah' blah1.txt
+  1:1:blah abc def
+
+Match a middle word:
+
+  $ ag -wF --column 'blah' blah2.txt
+  1:5:abc blah def
+
+Match a last word:
+
+  $ ag -wF --column 'blah' blah3.txt
+  1:9:abc def blah
+
+No match:
+
+  $ ag -wF --column 'blah' blah4.txt
+  [1]
+
+Match:
+
+  $ ag -wF --column 'blah' blah5.txt
+  1:7:blahx blah
+
+Case of a word repeating the same part:
+
+  $ ag -wF --column 'blah blah' blah6.txt
+  1:9:abcblah blah blah

tests/max_count.t

@@ -1,18 +1,18 @@
 Setup:
 
   $ . $TESTDIR/setup.sh
-  $ echo "blah" > blah.txt
-  $ echo "blah2" >> blah.txt
-  $ echo "blah2" > blah2.txt
-  $ echo "blah2" >> blah2.txt
-  $ echo "blah2" >> blah2.txt
-  $ echo "blah2" >> blah2.txt
-  $ echo "blah2" >> blah2.txt
-  $ echo "blah2" >> blah2.txt
-  $ echo "blah2" >> blah2.txt
-  $ echo "blah2" >> blah2.txt
-  $ echo "blah2" >> blah2.txt
-  $ echo "blah2" >> blah2.txt # 10 lines
+  $ printf "blah\n" > blah.txt
+  $ printf "blah2\n" >> blah.txt
+  $ printf "blah2\n" > blah2.txt
+  $ printf "blah2\n" >> blah2.txt
+  $ printf "blah2\n" >> blah2.txt
+  $ printf "blah2\n" >> blah2.txt
+  $ printf "blah2\n" >> blah2.txt
+  $ printf "blah2\n" >> blah2.txt
+  $ printf "blah2\n" >> blah2.txt
+  $ printf "blah2\n" >> blah2.txt
+  $ printf "blah2\n" >> blah2.txt
+  $ printf "blah2\n" >> blah2.txt # 10 lines
 
 Max match of 1:
 

tests/multiline.t

@@ -1,9 +1,9 @@
 Setup:
 
   $ . $TESTDIR/setup.sh
-  $ echo 'what' > blah.txt
-  $ echo 'ever' >> blah.txt
-  $ echo 'whatever' >> blah.txt
+  $ printf 'what\n' > blah.txt
+  $ printf 'ever\n' >> blah.txt
+  $ printf 'whatever\n' >> blah.txt
 
 Multiline:
 

tests/negated_options.t

@@ -0,0 +1,28 @@
+Setup:
+
+  $ . "${TESTDIR}/setup.sh"
+
+Should accept both --no-<option> and --no<option> forms.
+
+(Here we're just parsing out all of the options listed in the `ag` usage help
+that can be negated with 'no', and checking to make sure that each of them works
+with either form. This is slightly convoluted, but it should ensure that any
+options added in the future meet this requirement — assuming they're added to
+the usage help, anyway.)
+
+  $ test_negated_options() {
+  >   ag --help 2>&1 |
+  >   grep -oiE -- '--\[no-?\][a-z0-9_-]+' |
+  >   cut -d ']' -f '2' |
+  >   sort -u |
+  >   while read option; do
+  >     # The point here is that if the option we're testing is illegal, `ag`
+  >     # will catch it on invocation and dump the usage help, causing the test
+  >     # to produce output and thus fail
+  >     printf 'foo\n' | ag "--no-${option}" -v '^foo$' 2>&1
+  >     printf 'foo\n' | ag "--no${option}"  -v '^foo$' 2>&1
+  >   done
+  >   return 0
+  > }
+  $ test_negated_options
+

tests/one_device.t

@@ -4,9 +4,12 @@ Setup:
   > if [ ! -e "/dev/shm" ]; then
   > echo "No /dev/shm. Skipping test."
   > exit 80
+  > elif [ "$(stat -c%d /dev/)" = "$(stat -c%d /dev/shm/)" ]; then
+  > echo "/dev/shm not a different device.  Skipping test."
+  > exit 80
   > fi
   $ TEST_TMPDIR=`mktemp -d --tmpdir=/dev/shm ag_test.XXX`
-  $ echo "blah" > $TEST_TMPDIR/blah.txt
+  $ printf "blah\n" > $TEST_TMPDIR/blah.txt
   $ ln -s $TEST_TMPDIR other_device
 
 Should not descend into /dev/shm symlink when --one-device specified:

tests/only_matching.t

@@ -1,11 +1,11 @@
 Setup:
 
   $ . $TESTDIR/setup.sh
-  $ echo "the quick brown foxy" > blah.txt
-  $ echo "blah blah blah" >> blah.txt
-  $ echo "another foxlike word" >> blah.txt
-  $ echo "no matches here" >> blah.txt
-  $ echo "blah blah blah" >> blah.txt
+  $ printf "the quick brown foxy\n" > blah.txt
+  $ printf "blah blah blah\n" >> blah.txt
+  $ printf "another foxlike word\n" >> blah.txt
+  $ printf "no matches here\n" >> blah.txt
+  $ printf "blah blah blah\n" >> blah.txt
 
 Only print matches:
 

tests/option_smartcase.t

@@ -1,8 +1,8 @@
 Setup:
 
   $ . $TESTDIR/setup.sh
-  $ echo 'asdf' > test.txt
-  $ echo 'AsDf' >> test.txt
+  $ printf 'asdf\n' > test.txt
+  $ printf 'AsDf\n' >> test.txt
 
 Smart case search:
 

tests/passthrough.t

@@ -2,10 +2,10 @@ Setup:
 
   $ . $TESTDIR/setup.sh
   $ unalias ag
-  $ alias ag="$TESTDIR/../ag --nocolor --workers=1"
-  $ echo "foo bar" > passthrough_test.txt
-  $ echo "zoo zar" >> passthrough_test.txt
-  $ echo "foo test" >> passthrough_test.txt
+  $ alias ag="$TESTDIR/../ag --noaffinity --nocolor --workers=1"
+  $ printf "foo bar\n" > passthrough_test.txt
+  $ printf "zoo zar\n" >> passthrough_test.txt
+  $ printf "foo test\n" >> passthrough_test.txt
 
 No impact on non-stream:
 

tests/pipecontext.t

@@ -0,0 +1,29 @@
+Setup:
+
+  $ . $TESTDIR/setup.sh
+  $ mkdir pipecontext_dir
+  $ printf "a\nb\nc\n" > pipecontext_test.txt
+  $ cd pipecontext_dir
+
+Do not use parallel flag, which disables stream input:
+
+  $ unalias ag
+  $ alias ag="$TESTDIR/../ag --nocolor --workers=1"
+
+B flag on pipe:
+
+  $ cat ../pipecontext_test.txt | ag --numbers -B1 b
+  1-a
+  2:b
+
+C flag on pipe:
+
+  $ cat ../pipecontext_test.txt | ag --numbers -C1 b
+  1-a
+  2:b
+  3-c
+
+Just match last line:
+
+  $ cat ../pipecontext_test.txt | ag --numbers c
+  3:c

tests/print_all_files.t

@@ -0,0 +1,16 @@
+Setup:
+
+  $ . $TESTDIR/setup.sh
+  $ printf 'foo\n' > ./foo.txt
+  $ printf 'bar\n' > ./bar.txt
+  $ printf 'baz\n' > ./baz.txt
+
+All files:
+
+  $ ag --print-all-files --group foo | sort
+  
+  
+  1:foo
+  bar.txt
+  baz.txt
+  foo.txt

tests/print_end.t

@@ -0,0 +1,9 @@
+Setup:
+
+  $ . $TESTDIR/setup.sh
+  $ cp $TESTDIR/print_end.txt .
+
+Print match at the end of a file
+
+  $ ag ergneq1
+  print_end.txt:2:ergneq1

tests/print_end.txt

@@ -0,0 +1,2 @@
+ergneqergneqergneq
+ergneq1

tests/search_stdin.t

@@ -1,7 +1,7 @@
 Setup:
 
   $ . $TESTDIR/setup.sh
-  $ echo 'blah' > ./blah.txt
+  $ printf 'blah\n' > ./blah.txt
 
 Feed blah.txt from stdin:
 

tests/setup.sh

@@ -1,4 +1,9 @@
+#!/bin/bash
+
 # All cram tests should use this. Make sure that "ag" runs the version
 # of ag we just built, and make the output really simple.
 
-alias ag="$TESTDIR/../ag --nocolor --workers=1 --parallel"
+# --noaffinity is to stop Travis CI from erroring (it runs in containers so pthread_setaffinity_np fails)
+# --workers=1 is to keep all output ordered, to make testing output easier
+# shellcheck disable=2139
+alias ag="$TESTDIR/../ag --noaffinity --nocolor --workers=1 --parallel"

tests/stupid_fnmatch.t.disabled

@@ -2,8 +2,8 @@ Setup:
 
   $ . $TESTDIR/setup.sh
   $ mkdir -p ./a/bomb
-  $ echo 'whatever' > ./a/bomb/foo.yml
-  $ echo '*b/foo.yml' > ./.gitignore
+  $ printf 'whatever\n' > ./a/bomb/foo.yml
+  $ printf '*b/foo.yml\n' > ./.gitignore
 
 Ignore foo.yml but not blah.yml:
 

tests/vimgrep.t

@@ -1,10 +1,10 @@
 Setup:
 
   $ . $TESTDIR/setup.sh
-  $ echo 'Hello, "Hello, world" programs output "Hello, world".' > ./test_vimgrep.txt
-  $ echo '"Hello, world" programs are simple programs.' >> ./test_vimgrep.txt
-  $ echo 'They illustrate the most basic syntax of a programming language' >> ./test_vimgrep.txt
-  $ echo 'In javascript: alert("Hello, world!");' >> ./test_vimgrep.txt
+  $ printf 'Hello, "Hello, world" programs output "Hello, world".\n' > ./test_vimgrep.txt
+  $ printf '"Hello, world" programs are simple programs.\n' >> ./test_vimgrep.txt
+  $ printf 'They illustrate the most basic syntax of a programming language\n' >> ./test_vimgrep.txt
+  $ printf 'In javascript: alert("Hello, world!");\n' >> ./test_vimgrep.txt
 
 Search for lines matching "hello" in test_vimgrep.txt:
 

tests/word_regexp.t

@@ -0,0 +1,12 @@
+Setup:
+
+  $ . $TESTDIR/setup.sh
+  $ echo 'foo' > blah.txt
+  $ echo 'bar' >> blah.txt
+  $ echo 'foobar' >> blah.txt
+
+Word regexp:
+
+  $ ag -w 'foo|bar' ./
+  blah.txt:1:foo
+  blah.txt:2:bar

the_silver_searcher.spec.in

@@ -1,5 +1,5 @@
 %define _bashcompdir %_sysconfdir/bash_completion.d
-
+%define _zshcompdir %{_datadir}/zsh/site-functions
 
 Name:		the_silver_searcher
 Version:	@VERSION@
@@ -22,7 +22,7 @@ An attempt to make something better than ack (which itself is better than grep).
 Why use Ag?
 * It searches code about 3–5× faster than ack.
 * It ignores file patterns from your .gitignore and .hgignore.
-* If there are files in your source repo you don't want to search, just add their patterns to a .agignore file. *cough* extern *cough*
+* If there are files in your source repo you don't want to search, just add their patterns to a .ignore file. *cough* extern *cough*
 * The command name is 33% shorter than ack!
 
 How is it so fast?
@@ -62,7 +62,7 @@ rm -rf ${RPM_BUILD_ROOT}
 %{_mandir}/*
 %config %{_bashcompdir}/ag.bashcomp.sh
 %config %{_datadir}/%{name}/completions/ag.bashcomp.sh
-
+%config %{_datadir}/zsh/site-functions/_the_silver_searcher
 
 %changelog
 * Thu Dec 5 2013 Emily Strickland <code@emily.st> - 0.18.1-1