diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 86fa5e7..433337d 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,6 +1,6 @@ # These are supported funding model platforms -github: OJ +github: [OJ, firefart] patreon: OJReeves open_collective: gobuster ko_fi: OJReeves diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..c47c63f --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,17 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "gomod" + directory: "/" + schedule: + interval: "weekly" + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + # Check for updates to GitHub Actions every weekday + interval: "daily" diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 39aa874..c668bab 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go: ["1.17", "1.18", "1.19"] + go: ["1.18", "1.19"] steps: - name: Set up Go ${{ matrix.go }} uses: actions/setup-go@v2 @@ -28,8 +28,11 @@ jobs: run: | go get -v -t -d ./... - - name: Build - run: go build -v . + - name: Build linux + run: make linux + + - name: Build windows + run: make windows - name: Test run: make test diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..9276ead --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,32 @@ +name: goreleaser + +on: + push: + tags: + - "*" + +permissions: + contents: write + +jobs: + goreleaser: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch all tags + run: git fetch --force --tags + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.19 + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v2 + with: + distribution: goreleaser + version: latest + args: release --rm-dist + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 06160d6..1bd53f0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,26 +1,19 @@ -# Compiled Object files, Static and Dynamic libs (Shared Objects) -*.o -*.a -*.so - -# Folders -_obj -_test - -# Architecture specific extensions/prefixes -*.[568vq] -[568vq].out - -*.cgo1.go -*.cgo2.c -_cgo_defun.c -_cgo_gotypes.go -_cgo_export.* - -_testmain.go - +# Binaries for programs and plugins *.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` *.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + *.prof *.txt *.swp @@ -28,6 +21,5 @@ _testmain.go .vscode/ gobuster build -v3 -.idea/ +dist/ diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..6c6e4ba --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,3 @@ +linters: + enable: + - nonamedreturns diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..6b8cb77 --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,36 @@ +# This is an example .goreleaser.yml file with some sensible defaults. +# Make sure to check the documentation at https://goreleaser.com +before: + hooks: + # You may remove this if you don't use go modules. + - go mod tidy + # you may remove this if you don't need go generate + - go generate ./... +builds: + - env: + - CGO_ENABLED=0 + goos: + - linux + - windows + - darwin +archives: + - format: tar.gz + format_overrides: + - goos: windows + format: zip + replacements: + darwin: Darwin + linux: Linux + windows: Windows + 386: i386 + amd64: x86_64 +checksum: + name_template: "checksums.txt" +snapshot: + name_template: "{{ incpatch .Version }}-dev" +changelog: + sort: asc + filters: + exclude: + - "^docs:" + - "^test:" diff --git a/Makefile b/Makefile index bdbe480..33067ac 100644 --- a/Makefile +++ b/Makefile @@ -1,57 +1,28 @@ -TARGET=./build -ARCHS=amd64 386 -LDFLAGS="-s -w" - -.PHONY: current -current: - @go build -o ./gobuster; \ - echo "Done." - -.PHONY: fmt -fmt: - @go fmt ./...; \ - echo "Done." - -.PHONY: update -update: - @go get -u; \ - go mod tidy -v; \ - echo "Done." - -.PHONY: windows -windows: - @for GOARCH in ${ARCHS}; do \ - echo "Building for windows $${GOARCH} ..." ; \ - mkdir -p ${TARGET}/gobuster-windows-$${GOARCH} ; \ - GOOS=windows GOARCH=$${GOARCH} GO111MODULE=on CGO_ENABLED=0 go build -ldflags=${LDFLAGS} -trimpath -o ${TARGET}/gobuster-windows-$${GOARCH}/gobuster.exe ; \ - done; \ - echo "Done." +.DEFAULT_GOAL := linux .PHONY: linux linux: - @for GOARCH in ${ARCHS}; do \ - echo "Building for linux $${GOARCH} ..." ; \ - mkdir -p ${TARGET}/gobuster-linux-$${GOARCH} ; \ - GOOS=linux GOARCH=$${GOARCH} GO111MODULE=on CGO_ENABLED=0 go build -ldflags=${LDFLAGS} -trimpath -o ${TARGET}/gobuster-linux-$${GOARCH}/gobuster ; \ - done; \ - echo "Done." + go build -o ./gobuster -.PHONY: darwin -darwin: - @for GOARCH in ${ARCHS}; do \ - echo "Building for darwin $${GOARCH} ..." ; \ - mkdir -p ${TARGET}/gobuster-darwin-$${GOARCH} ; \ - GOOS=darwin GOARCH=$${GOARCH} GO111MODULE=on CGO_ENABLED=0 go build -ldflags=${LDFLAGS} -trimpath -o ${TARGET}/gobuster-darwin-$${GOARCH}/gobuster ; \ - done; \ - echo "Done." +.PHONY: windows +windows: + GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -o ./gobuster.exe + +.PHONY: fmt +fmt: + go fmt ./... + +.PHONY: update +update: + go get -u + go mod tidy -v .PHONY: all -all: clean fmt update test lint darwin linux windows +all: fmt update linux windows test lint .PHONY: test test: - @go test -v -race ./... ; \ - echo "Done." + go test -v -race ./... .PHONY: lint lint: @@ -62,14 +33,3 @@ lint: lint-update: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin $$(go env GOPATH)/bin/golangci-lint --version - -.PHONY: lint-docker -lint-docker: - docker pull golangci/golangci-lint:latest - docker run --rm -v $$(pwd):/app -w /app golangci/golangci-lint:latest golangci-lint run - -.PHONY: clean -clean: - @rm -rf ${TARGET}/* ; \ - go clean ./... ; \ - echo "Done." diff --git a/README.md b/README.md index 09b04e1..5e60339 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Gobuster v3.1.0 +# Gobuster v3.2.0 Gobuster is a tool used to brute-force: @@ -9,28 +9,8 @@ Gobuster is a tool used to brute-force: ## Tags, Statuses, etc -[![Build Status](https://travis-ci.com/OJ/gobuster.svg?branch=master)](https://travis-ci.com/OJ/gobuster) [![Backers on Open Collective](https://opencollective.com/gobuster/backers/badge.svg)] [![Sponsors on Open Collective](https://opencollective.com/gobuster/sponsors/badge.svg)] +[![Build Status](https://travis-ci.com/OJ/gobuster.svg?branch=master)](https://travis-ci.com/OJ/gobuster) [![Backers on Open Collective](https://opencollective.com/gobuster/backers/badge.svg)](https://opencollective.com/gobuster) [![Sponsors on Open Collective](https://opencollective.com/gobuster/sponsors/badge.svg)](https://opencollective.com/gobuster) -## Oh dear God.. WHY!? - -Because I wanted: - -1. ... something that didn't have a fat Java GUI (console FTW). -1. ... to build something that just worked on the command line. -1. ... something that did not do recursive brute force. -1. ... something that allowed me to brute force folders and multiple extensions at once. -1. ... something that compiled to native on multiple platforms. -1. ... something that was faster than an interpreted script (such as Python). -1. ... something that didn't require a runtime. -1. ... use something that was good with concurrency (hence Go). -1. ... to build something in Go that wasn't totally useless. - -## But it's shit! And your implementation sucks! - -Yes, you're probably correct. Feel free to: - -- Not use it. -- Show me how to do it better. ## Love this tool? Back it! @@ -40,13 +20,19 @@ If you're backing us already, you rock. If you're not, that's cool too! Want to All funds that are donated to this project will be donated to charity. A full log of charity donations will be available in this repository as they are processed. -## Changes in 3.1-dev +# Changes -- Use go 1.16 +## 3.2-dev + +- Use go 1.19 - use contexts in the correct way - get rid of the wildcard flag (except in DNS mode) +- color output +- retry on timeout +- google cloud bucket enumeration +- fix nil reference errors -## Changes in 3.1 +## 3.1 - enumerate public AWS S3 buckets - fuzzing mode @@ -54,118 +40,27 @@ All funds that are donated to this project will be donated to charity. A full lo - added support for patterns. You can now specify a file containing patterns that are applied to every word, one by line. Every occurrence of the term `{GOBUSTER}` in it will be replaced with the current wordlist item. Please use with caution as this can cause increase the number of requests issued a lot. - The shorthand `p` flag which was assigned to proxy is now used by the pattern flag -## Changes in 3.0 +## 3.0 - New CLI options so modes are strictly separated (`-m` is now gone!) - Performance Optimizations and better connection handling - Ability to enumerate vhost names - Option to supply custom HTTP headers +# License + +See the LICENSE file. + +# Manual + ## Available Modes - dir - the classic directory brute-forcing mode - dns - DNS subdomain brute-forcing mode - s3 - Enumerate open S3 buckets and look for existence and bucket listings +- gcs - Enumerate open google cloud buckets - vhost - virtual host brute-forcing mode (not the same as DNS!) - -## Built-in Help - -Help is built-in! - -- `gobuster help` - outputs the top-level help. -- `gobuster help ` - outputs the help specific to that mode. - -## `dns` Mode Help - -```text -Usage: - gobuster dns [flags] - -Flags: - -d, --domain string The target domain - -h, --help help for dns - -r, --resolver string Use custom DNS server (format server.com or server.com:port) - -c, --show-cname Show CNAME records (cannot be used with '-i' option) - -i, --show-ips Show IP addresses - --timeout duration DNS resolver timeout (default 1s) - --wildcard Force continued operation when wildcard found - -Global Flags: - -z, --no-progress Don't display progress - -o, --output string Output file to write results to (defaults to stdout) - -q, --quiet Don't print the banner and other noise - -t, --threads int Number of concurrent threads (default 10) - --delay duration Time each thread waits between requests (e.g. 1500ms) - -v, --verbose Verbose output (errors) - -w, --wordlist string Path to the wordlist -``` - -## `dir` Mode Options - -```text -Usage: - gobuster dir [flags] - -Flags: - -f, --add-slash Append / to each request - -c, --cookies string Cookies to use for the requests - -e, --expanded Expanded mode, print full URLs - -x, --extensions string File extension(s) to search for - -r, --follow-redirect Follow redirects - -H, --headers stringArray Specify HTTP headers, -H 'Header1: val1' -H 'Header2: val2' - -h, --help help for dir - -l, --include-length Include the length of the body in the output - -k, --no-tls-validation Skip TLS certificate verification - -n, --no-status Don't print status codes - -P, --password string Password for Basic Auth - -p, --proxy string Proxy to use for requests [http(s)://host:port] - -s, --status-codes string Positive status codes (will be overwritten with status-codes-blacklist if set) (default "200,204,301,302,307,401,403") - -b, --status-codes-blacklist string Negative status codes (will override status-codes if set) - --timeout duration HTTP Timeout (default 10s) - -u, --url string The target URL - -a, --useragent string Set the User-Agent string (default "gobuster/3.1.0") - -U, --username string Username for Basic Auth - -d, --discover-backup Upon finding a file search for backup files - --wildcard Force continued operation when wildcard found - -Global Flags: - -z, --no-progress Don't display progress - -o, --output string Output file to write results to (defaults to stdout) - -q, --quiet Don't print the banner and other noise - -t, --threads int Number of concurrent threads (default 10) - --delay duration Time each thread waits between requests (e.g. 1500ms) - -v, --verbose Verbose output (errors) - -w, --wordlist string Path to the wordlist -``` - -## `vhost` Mode Options - -```text -Usage: - gobuster vhost [flags] - -Flags: - -c, --cookies string Cookies to use for the requests - -r, --follow-redirect Follow redirects - -H, --headers stringArray Specify HTTP headers, -H 'Header1: val1' -H 'Header2: val2' - -h, --help help for vhost - -k, --no-tls-validation Skip TLS certificate verification - -P, --password string Password for Basic Auth - -p, --proxy string Proxy to use for requests [http(s)://host:port] - --timeout duration HTTP Timeout (default 10s) - -u, --url string The target URL - -a, --useragent string Set the User-Agent string (default "gobuster/3.1.0") - -U, --username string Username for Basic Auth - -Global Flags: - -z, --no-progress Don't display progress - -o, --output string Output file to write results to (defaults to stdout) - -q, --quiet Don't print the banner and other noise - -t, --threads int Number of concurrent threads (default 10) - --delay duration Time each thread waits between requests (e.g. 1500ms) - -v, --verbose Verbose output (errors) - -w, --wordlist string Path to the wordlist -``` +- fuzz - some basic fuzzing, replaces the `FUZZ` keyword ## Easy Installation @@ -177,17 +72,17 @@ If you're stupid enough to trust binaries that I've put together, you can downlo ### Using `go install` -If you have a [Go](https://golang.org/) environment ready to go (at least go 1.17), it's as easy as: +If you have a [Go](https://golang.org/) environment ready to go (at least go 1.19), it's as easy as: ```bash go install github.com/OJ/gobuster/v3@latest ``` -PS: You need at least go 1.17.0 to compile gobuster. +PS: You need at least go 1.19 to compile gobuster. -## Building From Source +### Building From Source -Since this tool is written in [Go](https://golang.org/) you need to install the Go language/compiler/etc. Full details of installation and set up can be found [on the Go language website](https://golang.org/doc/install). Once installed you have two options. You need at least go 1.17.0 to compile gobuster. +Since this tool is written in [Go](https://golang.org/) you need to install the Go language/compiler/etc. Full details of installation and set up can be found [on the Go language website](https://golang.org/doc/install). Once installed you have two options. You need at least go 1.19 to compile gobuster. ### Compiling @@ -203,196 +98,59 @@ This will create a `gobuster` binary for you. If you want to install it in the ` go install ``` -If you have all the dependencies already, you can make use of the build scripts: +## Modes -- `make` - builds for the current Go configuration (ie. runs `go build`). -- `make windows` - builds 32 and 64 bit binaries for windows, and writes them to the `build` folder. -- `make linux` - builds 32 and 64 bit binaries for linux, and writes them to the `build` folder. -- `make darwin` - builds 32 and 64 bit binaries for darwin, and writes them to the `build` folder. -- `make all` - builds for all platforms and architectures, and writes the resulting binaries to the `build` folder. -- `make clean` - clears out the `build` folder. -- `make test` - runs the tests. +Help is built-in! -## Wordlists via STDIN +- `gobuster help` - outputs the top-level help. +- `gobuster help ` - outputs the help specific to that mode. -Wordlists can be piped into `gobuster` via stdin by providing a `-` to the `-w` option: +## `dns` Mode -```bash -hashcat -a 3 --stdout ?l | gobuster dir -u https://mysite.com -w - -``` - -Note: If the `-w` option is specified at the same time as piping from STDIN, an error will be shown and the program will terminate. - -## Patterns - -You can supply pattern files that will be applied to every word from the wordlist. -Just place the string `{GOBUSTER}` in it and this will be replaced with the word. -This feature is also handy in s3 mode to pre- or postfix certain patterns. - -**Caution:** Using a big pattern file can cause a lot of request as every pattern is applied to every word in the wordlist. - -### Example file +### Options ```text -{GOBUSTER}Partial -{GOBUSTER}Service -PRE{GOBUSTER}POST -{GOBUSTER}-prod -{GOBUSTER}-dev +Uses DNS subdomain enumeration mode + +Usage: + gobuster dns [flags] + +Flags: + -d, --domain string The target domain + -h, --help help for dns + -r, --resolver string Use custom DNS server (format server.com or server.com:port) + -c, --show-cname Show CNAME records (cannot be used with '-i' option) + -i, --show-ips Show IP addresses + --timeout duration DNS resolver timeout (default 1s) + --wildcard Force continued operation when wildcard found + +Global Flags: + --delay duration Time each thread waits between requests (e.g. 1500ms) + --no-color Disable color output + --no-error Don't display errors + -z, --no-progress Don't display progress + -o, --output string Output file to write results to (defaults to stdout) + -p, --pattern string File containing replacement patterns + -q, --quiet Don't print the banner and other noise + -t, --threads int Number of concurrent threads (default 10) + -v, --verbose Verbose output (errors) + -w, --wordlist string Path to the wordlist ``` -## Examples +### Examples -### `dir` Mode -Command line might look like this: - -```bash -gobuster dir -u https://mysite.com/path/to/folder -c 'session=123456' -t 50 -w common-files.txt -x .php,.html -``` - -Default options looks like this: - -```bash -gobuster dir -u https://buffered.io -w ~/wordlists/shortlist.txt - -=============================================================== -Gobuster v3.1.0 -by OJ Reeves (@TheColonial) & Christian Mehlmauer (@firefart) -=============================================================== -[+] Mode : dir -[+] Url/Domain : https://buffered.io/ -[+] Threads : 10 -[+] Wordlist : /home/oj/wordlists/shortlist.txt -[+] Status codes : 200,204,301,302,307,401,403 -[+] User Agent : gobuster/3.1.0 -[+] Timeout : 10s -=============================================================== -2019/06/21 11:49:43 Starting gobuster -=============================================================== -/categories (Status: 301) -/contact (Status: 301) -/posts (Status: 301) -/index (Status: 200) -=============================================================== -2019/06/21 11:49:44 Finished -=============================================================== -``` - -Default options with status codes disabled looks like this: - -```bash -gobuster dir -u https://buffered.io -w ~/wordlists/shortlist.txt -n - -=============================================================== -Gobuster v3.1.0 -by OJ Reeves (@TheColonial) & Christian Mehlmauer (@firefart) -=============================================================== -[+] Mode : dir -[+] Url/Domain : https://buffered.io/ -[+] Threads : 10 -[+] Wordlist : /home/oj/wordlists/shortlist.txt -[+] Status codes : 200,204,301,302,307,401,403 -[+] User Agent : gobuster/3.1.0 -[+] No status : true -[+] Timeout : 10s -=============================================================== -2019/06/21 11:50:18 Starting gobuster -=============================================================== -/categories -/contact -/index -/posts -=============================================================== -2019/06/21 11:50:18 Finished -=============================================================== -``` - -Verbose output looks like this: - -```bash -gobuster dir -u https://buffered.io -w ~/wordlists/shortlist.txt -v - -=============================================================== -Gobuster v3.1.0 -by OJ Reeves (@TheColonial) & Christian Mehlmauer (@firefart) -=============================================================== -[+] Mode : dir -[+] Url/Domain : https://buffered.io/ -[+] Threads : 10 -[+] Wordlist : /home/oj/wordlists/shortlist.txt -[+] Status codes : 200,204,301,302,307,401,403 -[+] User Agent : gobuster/3.1.0 -[+] Verbose : true -[+] Timeout : 10s -=============================================================== -2019/06/21 11:50:51 Starting gobuster -=============================================================== -Missed: /alsodoesnotexist (Status: 404) -Found: /index (Status: 200) -Missed: /doesnotexist (Status: 404) -Found: /categories (Status: 301) -Found: /posts (Status: 301) -Found: /contact (Status: 301) -=============================================================== -2019/06/21 11:50:51 Finished -=============================================================== -``` - -Example showing content length: - -```bash -gobuster dir -u https://buffered.io -w ~/wordlists/shortlist.txt -l - -=============================================================== -Gobuster v3.1.0 -by OJ Reeves (@TheColonial) & Christian Mehlmauer (@firefart) -=============================================================== -[+] Mode : dir -[+] Url/Domain : https://buffered.io/ -[+] Threads : 10 -[+] Wordlist : /home/oj/wordlists/shortlist.txt -[+] Status codes : 200,204,301,302,307,401,403 -[+] User Agent : gobuster/3.1.0 -[+] Show length : true -[+] Timeout : 10s -=============================================================== -2019/06/21 11:51:16 Starting gobuster -=============================================================== -/categories (Status: 301) [Size: 178] -/posts (Status: 301) [Size: 178] -/contact (Status: 301) [Size: 178] -/index (Status: 200) [Size: 51759] -=============================================================== -2019/06/21 11:51:17 Finished -=============================================================== -``` - -Quiet output, with status disabled and expanded mode looks like this ("grep mode"): - -```bash -gobuster dir -u https://buffered.io -w ~/wordlists/shortlist.txt -q -n -e -https://buffered.io/index -https://buffered.io/contact -https://buffered.io/posts -https://buffered.io/categories -``` - -### `dns` Mode - -Command line might look like this: - -```bash +```text gobuster dns -d mysite.com -t 50 -w common-names.txt ``` Normal sample run goes like this: -```bash +```text gobuster dns -d google.com -w ~/wordlists/subdomains.txt =============================================================== -Gobuster v3.1.0 +Gobuster v3.2.0 by OJ Reeves (@TheColonial) & Christian Mehlmauer (@firefart) =============================================================== [+] Mode : dns @@ -427,11 +185,11 @@ Found: blog.google.com Show IP sample run goes like this: -```bash +```text gobuster dns -d google.com -w ~/wordlists/subdomains.txt -i =============================================================== -Gobuster v3.1.0 +Gobuster v3.2.0 by OJ Reeves (@TheColonial) & Christian Mehlmauer (@firefart) =============================================================== [+] Mode : dns @@ -466,11 +224,11 @@ Found: mail.google.com [172.217.25.37, 2404:6800:4006:802::2005] Base domain validation warning when the base domain fails to resolve. This is a warning rather than a failure in case the user fat-fingers while typing the domain. -```bash +```text gobuster dns -d yp.to -w ~/wordlists/subdomains.txt -i =============================================================== -Gobuster v3.1.0 +Gobuster v3.2.0 by OJ Reeves (@TheColonial) & Christian Mehlmauer (@firefart) =============================================================== [+] Mode : dns @@ -489,11 +247,11 @@ Found: cr.yp.to [131.193.32.108, 131.193.32.109] Wildcard DNS is also detected properly: -```bash +```text gobuster dns -d 0.0.1.xip.io -w ~/wordlists/subdomains.txt =============================================================== -Gobuster v3.1.0 +Gobuster v3.2.0 by OJ Reeves (@TheColonial) & Christian Mehlmauer (@firefart) =============================================================== [+] Mode : dns @@ -512,11 +270,11 @@ by OJ Reeves (@TheColonial) & Christian Mehlmauer (@firefart) If the user wants to force processing of a domain that has wildcard entries, use `--wildcard`: -```bash +```text gobuster dns -d 0.0.1.xip.io -w ~/wordlists/subdomains.txt --wildcard =============================================================== -Gobuster v3.1.0 +Gobuster v3.2.0 by OJ Reeves (@TheColonial) & Christian Mehlmauer (@firefart) =============================================================== [+] Mode : dns @@ -534,27 +292,251 @@ Found: test.127.0.0.1.xip.io =============================================================== ``` -### `vhost` Mode +## `dir` Mode -Command line might look like this: +### Options -```bash +```text +Uses directory/file enumeration mode + +Usage: + gobuster dir [flags] + +Flags: + -f, --add-slash Append / to each request + -c, --cookies string Cookies to use for the requests + -d, --discover-backup Also search for backup files by appending multiple backup extensions + --exclude-length ints exclude the following content length (completely ignores the status). Supply multiple times to exclude multiple sizes. + -e, --expanded Expanded mode, print full URLs + -x, --extensions string File extension(s) to search for + -r, --follow-redirect Follow redirects + -H, --headers stringArray Specify HTTP headers, -H 'Header1: val1' -H 'Header2: val2' + -h, --help help for dir + --hide-length Hide the length of the body in the output + -m, --method string Use the following HTTP method (default "GET") + -n, --no-status Don't print status codes + -k, --no-tls-validation Skip TLS certificate verification + -P, --password string Password for Basic Auth + --proxy string Proxy to use for requests [http(s)://host:port] + --random-agent Use a random User-Agent string + --retry Should retry on request timeout + --retry-attempts int Times to retry on request timeout (default 3) + -s, --status-codes string Positive status codes (will be overwritten with status-codes-blacklist if set) + -b, --status-codes-blacklist string Negative status codes (will override status-codes if set) (default "404") + --timeout duration HTTP Timeout (default 10s) + -u, --url string The target URL + -a, --useragent string Set the User-Agent string (default "gobuster/3.2.0") + -U, --username string Username for Basic Auth + +Global Flags: + --delay duration Time each thread waits between requests (e.g. 1500ms) + --no-color Disable color output + --no-error Don't display errors + -z, --no-progress Don't display progress + -o, --output string Output file to write results to (defaults to stdout) + -p, --pattern string File containing replacement patterns + -q, --quiet Don't print the banner and other noise + -t, --threads int Number of concurrent threads (default 10) + -v, --verbose Verbose output (errors) + -w, --wordlist string Path to the wordlist +``` + +### Examples + +```text +gobuster dir -u https://mysite.com/path/to/folder -c 'session=123456' -t 50 -w common-files.txt -x .php,.html +``` + +Default options looks like this: + +```text +gobuster dir -u https://buffered.io -w ~/wordlists/shortlist.txt + +=============================================================== +Gobuster v3.2.0 +by OJ Reeves (@TheColonial) & Christian Mehlmauer (@firefart) +=============================================================== +[+] Mode : dir +[+] Url/Domain : https://buffered.io/ +[+] Threads : 10 +[+] Wordlist : /home/oj/wordlists/shortlist.txt +[+] Status codes : 200,204,301,302,307,401,403 +[+] User Agent : gobuster/3.2.0 +[+] Timeout : 10s +=============================================================== +2019/06/21 11:49:43 Starting gobuster +=============================================================== +/categories (Status: 301) +/contact (Status: 301) +/posts (Status: 301) +/index (Status: 200) +=============================================================== +2019/06/21 11:49:44 Finished +=============================================================== +``` + +Default options with status codes disabled looks like this: + +```text +gobuster dir -u https://buffered.io -w ~/wordlists/shortlist.txt -n + +=============================================================== +Gobuster v3.2.0 +by OJ Reeves (@TheColonial) & Christian Mehlmauer (@firefart) +=============================================================== +[+] Mode : dir +[+] Url/Domain : https://buffered.io/ +[+] Threads : 10 +[+] Wordlist : /home/oj/wordlists/shortlist.txt +[+] Status codes : 200,204,301,302,307,401,403 +[+] User Agent : gobuster/3.2.0 +[+] No status : true +[+] Timeout : 10s +=============================================================== +2019/06/21 11:50:18 Starting gobuster +=============================================================== +/categories +/contact +/index +/posts +=============================================================== +2019/06/21 11:50:18 Finished +=============================================================== +``` + +Verbose output looks like this: + +```text +gobuster dir -u https://buffered.io -w ~/wordlists/shortlist.txt -v + +=============================================================== +Gobuster v3.2.0 +by OJ Reeves (@TheColonial) & Christian Mehlmauer (@firefart) +=============================================================== +[+] Mode : dir +[+] Url/Domain : https://buffered.io/ +[+] Threads : 10 +[+] Wordlist : /home/oj/wordlists/shortlist.txt +[+] Status codes : 200,204,301,302,307,401,403 +[+] User Agent : gobuster/3.2.0 +[+] Verbose : true +[+] Timeout : 10s +=============================================================== +2019/06/21 11:50:51 Starting gobuster +=============================================================== +Missed: /alsodoesnotexist (Status: 404) +Found: /index (Status: 200) +Missed: /doesnotexist (Status: 404) +Found: /categories (Status: 301) +Found: /posts (Status: 301) +Found: /contact (Status: 301) +=============================================================== +2019/06/21 11:50:51 Finished +=============================================================== +``` + +Example showing content length: + +```text +gobuster dir -u https://buffered.io -w ~/wordlists/shortlist.txt -l + +=============================================================== +Gobuster v3.2.0 +by OJ Reeves (@TheColonial) & Christian Mehlmauer (@firefart) +=============================================================== +[+] Mode : dir +[+] Url/Domain : https://buffered.io/ +[+] Threads : 10 +[+] Wordlist : /home/oj/wordlists/shortlist.txt +[+] Status codes : 200,204,301,302,307,401,403 +[+] User Agent : gobuster/3.2.0 +[+] Show length : true +[+] Timeout : 10s +=============================================================== +2019/06/21 11:51:16 Starting gobuster +=============================================================== +/categories (Status: 301) [Size: 178] +/posts (Status: 301) [Size: 178] +/contact (Status: 301) [Size: 178] +/index (Status: 200) [Size: 51759] +=============================================================== +2019/06/21 11:51:17 Finished +=============================================================== +``` + +Quiet output, with status disabled and expanded mode looks like this ("grep mode"): + +```text +gobuster dir -u https://buffered.io -w ~/wordlists/shortlist.txt -q -n -e +https://buffered.io/index +https://buffered.io/contact +https://buffered.io/posts +https://buffered.io/categories +``` + +## `vhost` Mode + +### Options + +```text +Uses VHOST enumeration mode (you most probably want to use the IP adress as the URL parameter + +Usage: + gobuster vhost [flags] + +Flags: + --append-domain Append main domain from URL to words from wordlist. Otherwise the fully qualified domains need to be specified in the wordlist. + -c, --cookies string Cookies to use for the requests + --domain string the domain to append when using an IP address as URL. If left empty and you specify a domain based URL the hostname from the URL is extracted + --exclude-length ints exclude the following content length (completely ignores the status). Supply multiple times to exclude multiple sizes. + -r, --follow-redirect Follow redirects + -H, --headers stringArray Specify HTTP headers, -H 'Header1: val1' -H 'Header2: val2' + -h, --help help for vhost + -m, --method string Use the following HTTP method (default "GET") + -k, --no-tls-validation Skip TLS certificate verification + -P, --password string Password for Basic Auth + --proxy string Proxy to use for requests [http(s)://host:port] + --random-agent Use a random User-Agent string + --retry Should retry on request timeout + --retry-attempts int Times to retry on request timeout (default 3) + --timeout duration HTTP Timeout (default 10s) + -u, --url string The target URL + -a, --useragent string Set the User-Agent string (default "gobuster/3.2.0") + -U, --username string Username for Basic Auth + +Global Flags: + --delay duration Time each thread waits between requests (e.g. 1500ms) + --no-color Disable color output + --no-error Don't display errors + -z, --no-progress Don't display progress + -o, --output string Output file to write results to (defaults to stdout) + -p, --pattern string File containing replacement patterns + -q, --quiet Don't print the banner and other noise + -t, --threads int Number of concurrent threads (default 10) + -v, --verbose Verbose output (errors) + -w, --wordlist string Path to the wordlist +``` + +### Examples + + +```text gobuster vhost -u https://mysite.com -w common-vhosts.txt ``` Normal sample run goes like this: -```bash +```text gobuster vhost -u https://mysite.com -w common-vhosts.txt =============================================================== -Gobuster v3.1.0 +Gobuster v3.2.0 by OJ Reeves (@TheColonial) & Christian Mehlmauer (@firefart) =============================================================== [+] Url: https://mysite.com [+] Threads: 10 [+] Wordlist: common-vhosts.txt -[+] User Agent: gobuster/3.1.0 +[+] User Agent: gobuster/3.2.0 [+] Timeout: 10s =============================================================== 2019/06/21 08:36:00 Starting gobuster @@ -567,20 +549,160 @@ Found: mail.mysite.com =============================================================== ``` -### `s3` Mode +## `fuzz` Mode -Command line might look like this: +### Options -```bash +```text +Uses fuzzing mode + +Usage: + gobuster fuzz [flags] + +Flags: + -c, --cookies string Cookies to use for the requests + --exclude-length ints exclude the following content length (completely ignores the status). Supply multiple times to exclude multiple sizes. + -b, --excludestatuscodes string Negative status codes (will override statuscodes if set) + -r, --follow-redirect Follow redirects + -H, --headers stringArray Specify HTTP headers, -H 'Header1: val1' -H 'Header2: val2' + -h, --help help for fuzz + -m, --method string Use the following HTTP method (default "GET") + -k, --no-tls-validation Skip TLS certificate verification + -P, --password string Password for Basic Auth + --proxy string Proxy to use for requests [http(s)://host:port] + --random-agent Use a random User-Agent string + --retry Should retry on request timeout + --retry-attempts int Times to retry on request timeout (default 3) + --timeout duration HTTP Timeout (default 10s) + -u, --url string The target URL + -a, --useragent string Set the User-Agent string (default "gobuster/3.2.0") + -U, --username string Username for Basic Auth + +Global Flags: + --delay duration Time each thread waits between requests (e.g. 1500ms) + --no-color Disable color output + --no-error Don't display errors + -z, --no-progress Don't display progress + -o, --output string Output file to write results to (defaults to stdout) + -p, --pattern string File containing replacement patterns + -q, --quiet Don't print the banner and other noise + -t, --threads int Number of concurrent threads (default 10) + -v, --verbose Verbose output (errors) + -w, --wordlist string Path to the wordlist +``` + +### Examples + +```text +gobuster fuzz -u https://example.com?FUZZ=test -w parameter-names.txt +``` + +## `s3` Mode + +### Options + +```text +Uses aws bucket enumeration mode + +Usage: + gobuster s3 [flags] + +Flags: + -h, --help help for s3 + -m, --maxfiles int max files to list when listing buckets (only shown in verbose mode) (default 5) + -k, --no-tls-validation Skip TLS certificate verification + --proxy string Proxy to use for requests [http(s)://host:port] + --random-agent Use a random User-Agent string + --retry Should retry on request timeout + --retry-attempts int Times to retry on request timeout (default 3) + --timeout duration HTTP Timeout (default 10s) + -a, --useragent string Set the User-Agent string (default "gobuster/3.2.0") + +Global Flags: + --delay duration Time each thread waits between requests (e.g. 1500ms) + --no-color Disable color output + --no-error Don't display errors + -z, --no-progress Don't display progress + -o, --output string Output file to write results to (defaults to stdout) + -p, --pattern string File containing replacement patterns + -q, --quiet Don't print the banner and other noise + -t, --threads int Number of concurrent threads (default 10) + -v, --verbose Verbose output (errors) + -w, --wordlist string Path to the wordlist +``` + +### Examples + +```text gobuster s3 -w bucket-names.txt ``` -### `fuzzing` Mode +## `gcs` Mode -Command line might look like this: +### Options + +```text +Uses gcs bucket enumeration mode + +Usage: + gobuster gcs [flags] + +Flags: + -h, --help help for gcs + -m, --maxfiles int max files to list when listing buckets (only shown in verbose mode) (default 5) + -k, --no-tls-validation Skip TLS certificate verification + --proxy string Proxy to use for requests [http(s)://host:port] + --random-agent Use a random User-Agent string + --retry Should retry on request timeout + --retry-attempts int Times to retry on request timeout (default 3) + --timeout duration HTTP Timeout (default 10s) + -a, --useragent string Set the User-Agent string (default "gobuster/3.2.0") + +Global Flags: + --delay duration Time each thread waits between requests (e.g. 1500ms) + --no-color Disable color output + --no-error Don't display errors + -z, --no-progress Don't display progress + -o, --output string Output file to write results to (defaults to stdout) + -p, --pattern string File containing replacement patterns + -q, --quiet Don't print the banner and other noise + -t, --threads int Number of concurrent threads (default 10) + -v, --verbose Verbose output (errors) + -w, --wordlist string Path to the wordlist +``` + +### Examples + +```text +gobuster gcs -w bucket-names.txt +``` + +## Wordlists via STDIN + +Wordlists can be piped into `gobuster` via stdin by providing a `-` to the `-w` option: ```bash -gobuster fuzz -u https://example.com?FUZZ=test -w parameter-names.txt +hashcat -a 3 --stdout ?l | gobuster dir -u https://mysite.com -w - +``` + +Note: If the `-w` option is specified at the same time as piping from STDIN, an error will be shown and the program will terminate. + +## Patterns + +You can supply pattern files that will be applied to every word from the wordlist. +Just place the string `{GOBUSTER}` in it and this will be replaced with the word. +This feature is also handy in s3 mode to pre- or postfix certain patterns. + +**Caution:** Using a big pattern file can cause a lot of request as every pattern is applied to every word in the wordlist. + +### Example file + +```text +{GOBUSTER}Partial +{GOBUSTER}Service +PRE{GOBUSTER}POST +{GOBUSTER}-prod +{GOBUSTER}-dev ``` #### Use case in combination with patterns @@ -594,7 +716,7 @@ curl -s --output - https://raw.githubusercontent.com/eth0izzle/bucket-stream/mas - Run gobuster with the custom input. Be sure to turn verbose mode on to see the bucket details -```bash +```text gobuster s3 --wordlist my.custom.wordlist -p patterns.txt -v ``` @@ -603,12 +725,12 @@ Normal sample run goes like this: ```text PS C:\Users\firefart\Documents\code\gobuster> .\gobuster.exe s3 --wordlist .\wordlist.txt =============================================================== -Gobuster v3.1.0 +Gobuster v3.2.0 by OJ Reeves (@TheColonial) & Christian Mehlmauer (@firefart) =============================================================== [+] Threads: 10 [+] Wordlist: .\wordlist.txt -[+] User Agent: gobuster/3.1.0 +[+] User Agent: gobuster/3.2.0 [+] Timeout: 10s [+] Maximum files to list: 5 =============================================================== @@ -632,12 +754,12 @@ Verbose and sample run ```text PS C:\Users\firefart\Documents\code\gobuster> .\gobuster.exe s3 --wordlist .\wordlist.txt -v =============================================================== -Gobuster v3.1.0 +Gobuster v3.2.0 by OJ Reeves (@TheColonial) & Christian Mehlmauer (@firefart) =============================================================== [+] Threads: 10 [+] Wordlist: .\wordlist.txt -[+] User Agent: gobuster/3.1.0 +[+] User Agent: gobuster/3.2.0 [+] Verbose: true [+] Timeout: 10s [+] Maximum files to list: 5 @@ -662,12 +784,12 @@ Extended sample run ```text PS C:\Users\firefart\Documents\code\gobuster> .\gobuster.exe s3 --wordlist .\wordlist.txt -e =============================================================== -Gobuster v3.1.0 +Gobuster v3.2.0 by OJ Reeves (@TheColonial) & Christian Mehlmauer (@firefart) =============================================================== [+] Threads: 10 [+] Wordlist: .\wordlist.txt -[+] User Agent: gobuster/3.1.0 +[+] User Agent: gobuster/3.2.0 [+] Timeout: 10s [+] Expanded: true [+] Maximum files to list: 5 @@ -686,11 +808,3 @@ http://localhost.s3.amazonaws.com/ 2019/08/12 21:48:38 Finished =============================================================== ``` - -## License - -See the LICENSE file. - -## Thanks - -See the THANKS file for people who helped out. diff --git a/TODO.md b/TODO.md deleted file mode 100644 index 4640904..0000000 --- a/TODO.md +++ /dev/null @@ -1 +0,0 @@ -# TODO diff --git a/cli/cmd/dir.go b/cli/cmd/dir.go index 2a4cf32..61157db 100644 --- a/cli/cmd/dir.go +++ b/cli/cmd/dir.go @@ -59,6 +59,8 @@ func parseDirOptions() (*libgobuster.Options, *gobusterdir.OptionsDir, error) { plugin.NoTLSValidation = httpOpts.NoTLSValidation plugin.Headers = httpOpts.Headers plugin.Method = httpOpts.Method + plugin.RetryOnTimeout = httpOpts.RetryOnTimeout + plugin.RetryAttempts = httpOpts.RetryAttempts plugin.Extensions, err = cmdDir.Flags().GetString("extensions") if err != nil { @@ -93,7 +95,8 @@ func parseDirOptions() (*libgobuster.Options, *gobusterdir.OptionsDir, error) { plugin.StatusCodesBlacklistParsed = ret3 if plugin.StatusCodes != "" && plugin.StatusCodesBlacklist != "" { - return nil, nil, fmt.Errorf("status-codes and status-codes-blacklist are both set, please set only one") + return nil, nil, fmt.Errorf("status-codes (%q) and status-codes-blacklist (%q) are both set - please set only one. status-codes-blacklist is set by default so you might want to disable it by supplying an empty string.", + plugin.StatusCodes, plugin.StatusCodesBlacklist) } if plugin.StatusCodes == "" && plugin.StatusCodesBlacklist == "" { @@ -151,7 +154,7 @@ func init() { cmdDir.Flags().BoolP("no-status", "n", false, "Don't print status codes") cmdDir.Flags().Bool("hide-length", false, "Hide the length of the body in the output") cmdDir.Flags().BoolP("add-slash", "f", false, "Append / to each request") - cmdDir.Flags().BoolP("discover-backup", "d", false, "Upon finding a file search for backup files") + cmdDir.Flags().BoolP("discover-backup", "d", false, "Also search for backup files by appending multiple backup extensions") cmdDir.Flags().IntSlice("exclude-length", []int{}, "exclude the following content length (completely ignores the status). Supply multiple times to exclude multiple sizes.") cmdDir.PersistentPreRun = func(cmd *cobra.Command, args []string) { diff --git a/cli/cmd/fuzz.go b/cli/cmd/fuzz.go index 3b26c2c..d746e6e 100644 --- a/cli/cmd/fuzz.go +++ b/cli/cmd/fuzz.go @@ -59,6 +59,8 @@ func parseFuzzOptions() (*libgobuster.Options, *gobusterfuzz.OptionsFuzz, error) plugin.NoTLSValidation = httpOpts.NoTLSValidation plugin.Headers = httpOpts.Headers plugin.Method = httpOpts.Method + plugin.RetryOnTimeout = httpOpts.RetryOnTimeout + plugin.RetryAttempts = httpOpts.RetryAttempts // blacklist will override the normal status codes plugin.ExcludedStatusCodes, err = cmdFuzz.Flags().GetString("excludestatuscodes") diff --git a/cli/cmd/gcs.go b/cli/cmd/gcs.go new file mode 100644 index 0000000..9be0cda --- /dev/null +++ b/cli/cmd/gcs.go @@ -0,0 +1,76 @@ +package cmd + +import ( + "fmt" + + "github.com/OJ/gobuster/v3/cli" + "github.com/OJ/gobuster/v3/gobustergcs" + "github.com/OJ/gobuster/v3/libgobuster" + "github.com/spf13/cobra" +) + +// nolint:gochecknoglobals +var cmdGCS *cobra.Command + +func runGCS(cmd *cobra.Command, args []string) error { + globalopts, pluginopts, err := parseGCSOptions() + if err != nil { + return fmt.Errorf("error on parsing arguments: %w", err) + } + + plugin, err := gobustergcs.NewGobusterGCS(globalopts, pluginopts) + if err != nil { + return fmt.Errorf("error on creating gobustergcs: %w", err) + } + + if err := cli.Gobuster(mainContext, globalopts, plugin); err != nil { + return fmt.Errorf("error on running gobuster: %w", err) + } + return nil +} + +func parseGCSOptions() (*libgobuster.Options, *gobustergcs.OptionsGCS, error) { + globalopts, err := parseGlobalOptions() + if err != nil { + return nil, nil, err + } + + pluginopts := gobustergcs.NewOptionsGCS() + + httpOpts, err := parseBasicHTTPOptions(cmdGCS) + if err != nil { + return nil, nil, err + } + + pluginopts.UserAgent = httpOpts.UserAgent + pluginopts.Proxy = httpOpts.Proxy + pluginopts.Timeout = httpOpts.Timeout + pluginopts.NoTLSValidation = httpOpts.NoTLSValidation + pluginopts.RetryOnTimeout = httpOpts.RetryOnTimeout + pluginopts.RetryAttempts = httpOpts.RetryAttempts + + pluginopts.MaxFilesToList, err = cmdGCS.Flags().GetInt("maxfiles") + if err != nil { + return nil, nil, fmt.Errorf("invalid value for maxfiles: %w", err) + } + + return globalopts, pluginopts, nil +} + +// nolint:gochecknoinits +func init() { + cmdGCS = &cobra.Command{ + Use: "gcs", + Short: "Uses gcs bucket enumeration mode", + RunE: runGCS, + } + + addBasicHTTPOptions(cmdGCS) + cmdGCS.Flags().IntP("maxfiles", "m", 5, "max files to list when listing buckets (only shown in verbose mode)") + + cmdGCS.PersistentPreRun = func(cmd *cobra.Command, args []string) { + configureGlobalOptions() + } + + rootCmd.AddCommand(cmdGCS) +} diff --git a/cli/cmd/http.go b/cli/cmd/http.go index 1a88a73..65c3ecb 100644 --- a/cli/cmd/http.go +++ b/cli/cmd/http.go @@ -20,6 +20,8 @@ func addBasicHTTPOptions(cmd *cobra.Command) { cmd.Flags().StringP("proxy", "", "", "Proxy to use for requests [http(s)://host:port]") cmd.Flags().DurationP("timeout", "", 10*time.Second, "HTTP Timeout") cmd.Flags().BoolP("no-tls-validation", "k", false, "Skip TLS certificate verification") + cmd.Flags().BoolP("retry", "", false, "Should retry on request timeout") + cmd.Flags().IntP("retry-attempts", "", 3, "Times to retry on request timeout") } func addCommonHTTPOptions(cmd *cobra.Command) error { @@ -69,6 +71,16 @@ func parseBasicHTTPOptions(cmd *cobra.Command) (libgobuster.BasicHTTPOptions, er return options, fmt.Errorf("invalid value for timeout: %w", err) } + options.RetryOnTimeout, err = cmd.Flags().GetBool("retry") + if err != nil { + return options, fmt.Errorf("invalid value for retry: %w", err) + } + + options.RetryAttempts, err = cmd.Flags().GetInt("retry-attempts") + if err != nil { + return options, fmt.Errorf("invalid value for retry-attempts: %w", err) + } + options.NoTLSValidation, err = cmd.Flags().GetBool("no-tls-validation") if err != nil { return options, fmt.Errorf("invalid value for no-tls-validation: %w", err) @@ -88,6 +100,8 @@ func parseCommonHTTPOptions(cmd *cobra.Command) (libgobuster.HTTPOptions, error) options.Timeout = basic.Timeout options.UserAgent = basic.UserAgent options.NoTLSValidation = basic.NoTLSValidation + options.RetryOnTimeout = basic.RetryOnTimeout + options.RetryAttempts = basic.RetryAttempts options.URL, err = cmd.Flags().GetString("url") if err != nil { diff --git a/cli/cmd/root.go b/cli/cmd/root.go index 6d2e534..ef95ecb 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -9,6 +9,7 @@ import ( "os/signal" "github.com/OJ/gobuster/v3/libgobuster" + "github.com/fatih/color" "github.com/spf13/cobra" ) @@ -136,6 +137,14 @@ func parseGlobalOptions() (*libgobuster.Options, error) { return nil, fmt.Errorf("invalid value for no-error: %w", err) } + noColor, err := rootCmd.Flags().GetBool("no-color") + if err != nil { + return nil, fmt.Errorf("invalid value for no-color: %w", err) + } + if noColor { + color.NoColor = true + } + return globalopts, nil } @@ -160,4 +169,5 @@ func init() { rootCmd.PersistentFlags().BoolP("no-progress", "z", false, "Don't display progress") rootCmd.PersistentFlags().Bool("no-error", false, "Don't display errors") rootCmd.PersistentFlags().StringP("pattern", "p", "", "File containing replacement patterns") + rootCmd.PersistentFlags().Bool("no-color", false, "Disable color output") } diff --git a/cli/cmd/s3.go b/cli/cmd/s3.go index 5e75e1a..5477b88 100644 --- a/cli/cmd/s3.go +++ b/cli/cmd/s3.go @@ -46,6 +46,8 @@ func parseS3Options() (*libgobuster.Options, *gobusters3.OptionsS3, error) { plugin.Proxy = httpOpts.Proxy plugin.Timeout = httpOpts.Timeout plugin.NoTLSValidation = httpOpts.NoTLSValidation + plugin.RetryOnTimeout = httpOpts.RetryOnTimeout + plugin.RetryAttempts = httpOpts.RetryAttempts plugin.MaxFilesToList, err = cmdS3.Flags().GetInt("maxfiles") if err != nil { diff --git a/cli/cmd/vhost.go b/cli/cmd/vhost.go index 0dc2f5f..ac047bd 100644 --- a/cli/cmd/vhost.go +++ b/cli/cmd/vhost.go @@ -52,6 +52,8 @@ func parseVhostOptions() (*libgobuster.Options, *gobustervhost.OptionsVhost, err plugin.NoTLSValidation = httpOpts.NoTLSValidation plugin.Headers = httpOpts.Headers plugin.Method = httpOpts.Method + plugin.RetryOnTimeout = httpOpts.RetryOnTimeout + plugin.RetryAttempts = httpOpts.RetryAttempts plugin.AppendDomain, err = cmdVhost.Flags().GetBool("append-domain") if err != nil { diff --git a/cli/const.go b/cli/const.go new file mode 100644 index 0000000..f9cea13 --- /dev/null +++ b/cli/const.go @@ -0,0 +1,7 @@ +//go:build !windows + +package cli + +const ( + TERMINAL_CLEAR_LINE = "\r\x1b[2K" +) diff --git a/cli/const_windows.go b/cli/const_windows.go new file mode 100644 index 0000000..ed913fd --- /dev/null +++ b/cli/const_windows.go @@ -0,0 +1,7 @@ +//go:build windows + +package cli + +const ( + TERMINAL_CLEAR_LINE = "\r\r" +) diff --git a/cli/gobuster.go b/cli/gobuster.go index 8bb509b..0918ea6 100644 --- a/cli/gobuster.go +++ b/cli/gobuster.go @@ -19,27 +19,9 @@ func banner() { fmt.Println("by OJ Reeves (@TheColonial) & Christian Mehlmauer (@firefart)") } -type outputType struct { - Mu *sync.RWMutex - MaxCharsWritten int -} - -// right pad a string -// nolint:unparam -func rightPad(s string, padStr string, overallLen int) string { - strLen := len(s) - if overallLen <= strLen { - return s - } - - toPad := overallLen - strLen - 1 - pad := strings.Repeat(padStr, toPad) - return fmt.Sprintf("%s%s", s, pad) -} - // resultWorker outputs the results as they come in. This needs to be a range and should not handle // the context so the channel always has a receiver and libgobuster will not block. -func resultWorker(g *libgobuster.Gobuster, filename string, wg *sync.WaitGroup, output *outputType) { +func resultWorker(g *libgobuster.Gobuster, filename string, wg *sync.WaitGroup) { defer wg.Done() var f *os.File @@ -52,20 +34,14 @@ func resultWorker(g *libgobuster.Gobuster, filename string, wg *sync.WaitGroup, defer f.Close() } - for r := range g.Results() { + for r := range g.Progress.ResultChan { s, err := r.ResultToString() if err != nil { g.LogError.Fatal(err) } if s != "" { s = strings.TrimSpace(s) - output.Mu.Lock() - w, _ := fmt.Printf("\r%s\n", rightPad(s, " ", output.MaxCharsWritten)) - // -1 to remove the newline, otherwise it's always bigger - if (w - 1) > output.MaxCharsWritten { - output.MaxCharsWritten = w - 1 - } - output.Mu.Unlock() + _, _ = fmt.Printf("%s%s\n", TERMINAL_CLEAR_LINE, s) if f != nil { err = writeToFile(f, s) if err != nil { @@ -78,21 +54,19 @@ func resultWorker(g *libgobuster.Gobuster, filename string, wg *sync.WaitGroup, // errorWorker outputs the errors as they come in. This needs to be a range and should not handle // the context so the channel always has a receiver and libgobuster will not block. -func errorWorker(g *libgobuster.Gobuster, wg *sync.WaitGroup, output *outputType) { +func errorWorker(g *libgobuster.Gobuster, wg *sync.WaitGroup) { defer wg.Done() - for e := range g.Errors() { + for e := range g.Progress.ErrorChan { if !g.Opts.Quiet && !g.Opts.NoError { - output.Mu.Lock() - g.LogError.Printf("[!] %v", e) - output.Mu.Unlock() + g.LogError.Printf("[!] %s\n", e.Error()) } } } // progressWorker outputs the progress every tick. It will stop once cancel() is called // on the context -func progressWorker(ctx context.Context, g *libgobuster.Gobuster, wg *sync.WaitGroup, output *outputType) { +func progressWorker(ctx context.Context, g *libgobuster.Gobuster, wg *sync.WaitGroup) { defer wg.Done() tick := time.NewTicker(cliProgressUpdate) @@ -101,25 +75,16 @@ func progressWorker(ctx context.Context, g *libgobuster.Gobuster, wg *sync.WaitG select { case <-tick.C: if !g.Opts.Quiet && !g.Opts.NoProgress { - g.RequestsCountMutex.RLock() - output.Mu.Lock() - var charsWritten int + requestsIssued := g.Progress.RequestsIssued() + requestsExpected := g.Progress.RequestsExpected() if g.Opts.Wordlist == "-" { - s := fmt.Sprintf("\rProgress: %d", g.RequestsIssued) - s = rightPad(s, " ", output.MaxCharsWritten) - charsWritten, _ = fmt.Fprint(os.Stderr, s) + s := fmt.Sprintf("%sProgress: %d", TERMINAL_CLEAR_LINE, requestsIssued) + _, _ = fmt.Fprint(os.Stderr, s) // only print status if we already read in the wordlist - } else if g.RequestsExpected > 0 { - s := fmt.Sprintf("\rProgress: %d / %d (%3.2f%%)", g.RequestsIssued, g.RequestsExpected, float32(g.RequestsIssued)*100.0/float32(g.RequestsExpected)) - s = rightPad(s, " ", output.MaxCharsWritten) - charsWritten, _ = fmt.Fprint(os.Stderr, s) + } else if requestsExpected > 0 { + s := fmt.Sprintf("%sProgress: %d / %d (%3.2f%%)", TERMINAL_CLEAR_LINE, requestsIssued, requestsExpected, float32(requestsIssued)*100.0/float32(requestsExpected)) + _, _ = fmt.Fprint(os.Stderr, s) } - if charsWritten > output.MaxCharsWritten { - output.MaxCharsWritten = charsWritten - } - - output.Mu.Unlock() - g.RequestsCountMutex.RUnlock() } case <-ctx.Done(): return @@ -173,22 +138,16 @@ func Gobuster(ctx context.Context, opts *libgobuster.Options, plugin libgobuster // when we call wg.Wait() var wg sync.WaitGroup - outputMutex := new(sync.RWMutex) - o := &outputType{ - Mu: outputMutex, - MaxCharsWritten: 0, - } + wg.Add(1) + go resultWorker(gobuster, opts.OutputFilename, &wg) wg.Add(1) - go resultWorker(gobuster, opts.OutputFilename, &wg, o) - - wg.Add(1) - go errorWorker(gobuster, &wg, o) + go errorWorker(gobuster, &wg) if !opts.Quiet && !opts.NoProgress { // if not quiet add a new workgroup entry and start the goroutine wg.Add(1) - go progressWorker(ctxCancel, gobuster, &wg, o) + go progressWorker(ctxCancel, gobuster, &wg) } err = gobuster.Run(ctxCancel) @@ -205,8 +164,6 @@ func Gobuster(ctx context.Context, opts *libgobuster.Options, plugin libgobuster } if !opts.Quiet { - // clear stderr progress - fmt.Fprintf(os.Stderr, "\r%s\n", rightPad("", " ", o.MaxCharsWritten)) fmt.Println(ruler) gobuster.LogInfo.Println("Finished") fmt.Println(ruler) diff --git a/cspell.json b/cspell.json index b9db312..25acbc1 100644 --- a/cspell.json +++ b/cspell.json @@ -1,7 +1,7 @@ // cSpell Settings { - // Version of the setting file. Always 0.1 - "version": "0.1", + // Version of the setting file. Always 0.2 + "version": "0.2", // language - current active spelling language "language": "en", // words - list of words to be always considered correct @@ -13,6 +13,7 @@ "gobusterdns", "gobusterfuzz", "gobustervhost", + "gobustergcs", "vhost", "vhosts", "cname", @@ -28,7 +29,10 @@ "unconvert", "unparam", "prealloc", - "gochecknoglobals" + "gochecknoglobals", + "gochecknoinits", + "fatih", + "netip" ], // flagWords - list of words to be always considered incorrect // This is useful for offensive words and common spelling errors. diff --git a/go.mod b/go.mod index 61dacc5..a6633a5 100644 --- a/go.mod +++ b/go.mod @@ -1,15 +1,18 @@ module github.com/OJ/gobuster/v3 +go 1.19 + require ( + github.com/fatih/color v1.13.0 github.com/google/uuid v1.3.0 github.com/spf13/cobra v1.5.0 - golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 + golang.org/x/term v0.0.0-20220919170432-7a66f970e087 ) require ( github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.16 // indirect github.com/spf13/pflag v1.0.5 // indirect - golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2 // indirect + golang.org/x/sys v0.0.0-20221006211917-84dc82d7e875 // indirect ) - -go 1.17 diff --git a/go.sum b/go.sum index d6ad755..1206708 100644 --- a/go.sum +++ b/go.sum @@ -1,18 +1,30 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2 h1:wM1k/lXfpc5HdkJJyW9GELpd8ERGdnh8sMGL6Gzq3Ho= -golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 h1:Q5284mrmYTpACcm+eAKjKJH48BBwSyfJqmmGDTtT8Vc= -golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221006211917-84dc82d7e875 h1:AzgQNqF+FKwyQ5LbVrVqOcuuFB67N47F9+htZYH0wFM= +golang.org/x/sys v0.0.0-20221006211917-84dc82d7e875/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20220919170432-7a66f970e087 h1:tPwmk4vmvVCMdr98VgL4JH+qZxPL8fqlUOHnyOM8N3w= +golang.org/x/term v0.0.0-20220919170432-7a66f970e087/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/gobusterdir/gobusterdir.go b/gobusterdir/gobusterdir.go index 60e4ae0..a823f38 100644 --- a/gobusterdir/gobusterdir.go +++ b/gobusterdir/gobusterdir.go @@ -5,6 +5,8 @@ import ( "bytes" "context" "fmt" + "net" + "net/http" "strings" "text/tabwriter" @@ -33,10 +35,9 @@ func (e *ErrWildcard) Error() string { // GobusterDir is the main type to implement the interface type GobusterDir struct { - options *OptionsDir - globalopts *libgobuster.Options - http *libgobuster.HTTPClient - requestsPerRun *int // helper variable so we do not recalculate this over and over + options *OptionsDir + globalopts *libgobuster.Options + http *libgobuster.HTTPClient } // NewGobusterDir creates a new initialized GobusterDir @@ -59,6 +60,8 @@ func NewGobusterDir(globalopts *libgobuster.Options, opts *OptionsDir) (*Gobuste Timeout: opts.Timeout, UserAgent: opts.UserAgent, NoTLSValidation: opts.NoTLSValidation, + RetryOnTimeout: opts.RetryOnTimeout, + RetryAttempts: opts.RetryAttempts, } httpOpts := libgobuster.HTTPOptions{ @@ -85,26 +88,6 @@ func (d *GobusterDir) Name() string { return "directory enumeration" } -// RequestsPerRun returns the number of requests this plugin makes per single wordlist item -func (d *GobusterDir) RequestsPerRun() int { - if d.requestsPerRun != nil { - return *d.requestsPerRun - } - - num := 1 + len(d.options.ExtensionsParsed.Set) - if d.options.DiscoverBackup { - // default word - num += len(backupExtensions) - num += len(backupDotExtensions) - // backups of filenames - num += len(d.options.ExtensionsParsed.Set) * len(backupExtensions) - num += len(d.options.ExtensionsParsed.Set) * len(backupDotExtensions) - } - d.requestsPerRun = &num - - return *d.requestsPerRun -} - // PreRun is the pre run implementation of gobusterdir func (d *GobusterDir) PreRun(ctx context.Context) error { // add trailing slash @@ -134,12 +117,12 @@ func (d *GobusterDir) PreRun(ctx context.Context) error { } if d.options.StatusCodesBlacklistParsed.Length() > 0 { - if !d.options.StatusCodesBlacklistParsed.Contains(*wildcardResp) { - return &ErrWildcard{url: url, statusCode: *wildcardResp, length: wildcardLength} + if !d.options.StatusCodesBlacklistParsed.Contains(wildcardResp) { + return &ErrWildcard{url: url, statusCode: wildcardResp, length: wildcardLength} } } else if d.options.StatusCodesParsed.Length() > 0 { - if d.options.StatusCodesParsed.Contains(*wildcardResp) { - return &ErrWildcard{url: url, statusCode: *wildcardResp, length: wildcardLength} + if d.options.StatusCodesParsed.Contains(wildcardResp) { + return &ErrWildcard{url: url, statusCode: wildcardResp, length: wildcardLength} } } else { return fmt.Errorf("StatusCodes and StatusCodesBlacklist are both not set which should not happen") @@ -163,72 +146,90 @@ func getBackupFilenames(word string) []string { return ret } -// Run is the process implementation of gobusterdir -func (d *GobusterDir) Run(ctx context.Context, word string, resChannel chan<- libgobuster.Result) error { - suffix := "" - if d.options.UseSlash { - suffix = "/" - } - +func (d *GobusterDir) AdditionalWords(word string) []string { + var words []string // build list of urls to check // 1: No extension // 2: With extension // 3: backupextension - urlsToCheck := make(map[string]string) - entity := fmt.Sprintf("%s%s", word, suffix) - dirURL := fmt.Sprintf("%s%s", d.options.URL, entity) - urlsToCheck[entity] = dirURL if d.options.DiscoverBackup { - for _, u := range getBackupFilenames(word) { - url := fmt.Sprintf("%s%s", d.options.URL, u) - urlsToCheck[u] = url - } + words = append(words, getBackupFilenames(word)...) } for ext := range d.options.ExtensionsParsed.Set { filename := fmt.Sprintf("%s.%s", word, ext) - url := fmt.Sprintf("%s%s", d.options.URL, filename) - urlsToCheck[filename] = url + words = append(words, filename) if d.options.DiscoverBackup { - for _, u := range getBackupFilenames(filename) { - url2 := fmt.Sprintf("%s%s", d.options.URL, u) - urlsToCheck[u] = url2 - } + words = append(words, getBackupFilenames(filename)...) } } + return words +} - for entity, url := range urlsToCheck { - statusCode, size, header, _, err := d.http.Request(ctx, url, libgobuster.RequestOptions{}) +// ProcessWord is the process implementation of gobusterdir +func (d *GobusterDir) ProcessWord(ctx context.Context, word string, progress *libgobuster.Progress) error { + suffix := "" + if d.options.UseSlash { + suffix = "/" + } + entity := fmt.Sprintf("%s%s", word, suffix) + url := fmt.Sprintf("%s%s", d.options.URL, entity) + + tries := 1 + if d.options.RetryOnTimeout && d.options.RetryAttempts > 0 { + // add it so it will be the overall max requests + tries += d.options.RetryAttempts + } + + var statusCode int + var size int64 + var header http.Header + for i := 1; i <= tries; i++ { + var err error + statusCode, size, header, _, err = d.http.Request(ctx, url, libgobuster.RequestOptions{}) if err != nil { - return err - } - if statusCode != nil { - resultStatus := false - - if d.options.StatusCodesBlacklistParsed.Length() > 0 { - if !d.options.StatusCodesBlacklistParsed.Contains(*statusCode) { - resultStatus = true - } - } else if d.options.StatusCodesParsed.Length() > 0 { - if d.options.StatusCodesParsed.Contains(*statusCode) { - resultStatus = true - } + // check if it's a timeout and if we should try again and try again + // otherwise the timeout error is raised + if netErr, ok := err.(net.Error); ok && netErr.Timeout() && i != tries { + continue + } else if strings.Contains(err.Error(), "invalid control character in URL") { + // put error in error chan so it's printed out and ignore it + // so gobuster will not quit + progress.ErrorChan <- err + continue } else { - return fmt.Errorf("StatusCodes and StatusCodesBlacklist are both not set which should not happen") + return err } + } + break + } - if (resultStatus && !helper.SliceContains(d.options.ExcludeLength, int(size))) || d.globalopts.Verbose { - resChannel <- Result{ - URL: d.options.URL, - Path: entity, - Verbose: d.globalopts.Verbose, - Expanded: d.options.Expanded, - NoStatus: d.options.NoStatus, - HideLength: d.options.HideLength, - Found: resultStatus, - Header: header, - StatusCode: *statusCode, - Size: size, - } + if statusCode != 0 { + resultStatus := false + + if d.options.StatusCodesBlacklistParsed.Length() > 0 { + if !d.options.StatusCodesBlacklistParsed.Contains(statusCode) { + resultStatus = true + } + } else if d.options.StatusCodesParsed.Length() > 0 { + if d.options.StatusCodesParsed.Contains(statusCode) { + resultStatus = true + } + } else { + return fmt.Errorf("StatusCodes and StatusCodesBlacklist are both not set which should not happen") + } + + if (resultStatus && !helper.SliceContains(d.options.ExcludeLength, int(size))) || d.globalopts.Verbose { + progress.ResultChan <- Result{ + URL: d.options.URL, + Path: entity, + Verbose: d.globalopts.Verbose, + Expanded: d.options.Expanded, + NoStatus: d.options.NoStatus, + HideLength: d.options.HideLength, + Found: resultStatus, + Header: header, + StatusCode: statusCode, + Size: size, } } } diff --git a/gobusterdir/options.go b/gobusterdir/options.go index 7c17136..93e821e 100644 --- a/gobusterdir/options.go +++ b/gobusterdir/options.go @@ -8,11 +8,11 @@ import ( type OptionsDir struct { libgobuster.HTTPOptions Extensions string - ExtensionsParsed libgobuster.StringSet + ExtensionsParsed libgobuster.Set[string] StatusCodes string - StatusCodesParsed libgobuster.IntSet + StatusCodesParsed libgobuster.Set[int] StatusCodesBlacklist string - StatusCodesBlacklistParsed libgobuster.IntSet + StatusCodesBlacklistParsed libgobuster.Set[int] UseSlash bool HideLength bool Expanded bool @@ -24,8 +24,8 @@ type OptionsDir struct { // NewOptionsDir returns a new initialized OptionsDir func NewOptionsDir() *OptionsDir { return &OptionsDir{ - StatusCodesParsed: libgobuster.NewIntSet(), - StatusCodesBlacklistParsed: libgobuster.NewIntSet(), - ExtensionsParsed: libgobuster.NewStringSet(), + StatusCodesParsed: libgobuster.NewSet[int](), + StatusCodesBlacklistParsed: libgobuster.NewSet[int](), + ExtensionsParsed: libgobuster.NewSet[string](), } } diff --git a/gobusterdir/result.go b/gobusterdir/result.go index 903a661..0abda21 100644 --- a/gobusterdir/result.go +++ b/gobusterdir/result.go @@ -4,6 +4,17 @@ import ( "bytes" "fmt" "net/http" + + "github.com/fatih/color" +) + +var ( + white = color.New(color.FgWhite).FprintfFunc() + yellow = color.New(color.FgYellow).FprintfFunc() + green = color.New(color.FgGreen).FprintfFunc() + blue = color.New(color.FgBlue).FprintfFunc() + red = color.New(color.FgRed).FprintfFunc() + cyan = color.New(color.FgCyan).FprintfFunc() ) // Result represents a single result @@ -51,9 +62,18 @@ func (r Result) ResultToString() (string, error) { } if !r.NoStatus { - if _, err := fmt.Fprintf(buf, " (Status: %d)", r.StatusCode); err != nil { - return "", err + color := white + if r.StatusCode == 200 { + color = green + } else if r.StatusCode >= 300 && r.StatusCode < 400 { + color = cyan + } else if r.StatusCode >= 400 && r.StatusCode < 500 { + color = yellow + } else if r.StatusCode >= 500 && r.StatusCode < 600 { + color = red } + + color(buf, " (Status: %d)", r.StatusCode) } if !r.HideLength { @@ -64,9 +84,7 @@ func (r Result) ResultToString() (string, error) { location := r.Header.Get("Location") if location != "" { - if _, err := fmt.Fprintf(buf, " [--> %s]", location); err != nil { - return "", err - } + blue(buf, " [--> %s]", location) } if _, err := fmt.Fprintf(buf, "\n"); err != nil { diff --git a/gobusterdns/gobusterdns.go b/gobusterdns/gobusterdns.go index b883fc1..3d9e622 100644 --- a/gobusterdns/gobusterdns.go +++ b/gobusterdns/gobusterdns.go @@ -7,6 +7,7 @@ import ( "fmt" "log" "net" + "net/netip" "strings" "text/tabwriter" "time" @@ -17,7 +18,7 @@ import ( // ErrWildcard is returned if a wildcard response is found type ErrWildcard struct { - wildcardIps libgobuster.StringSet + wildcardIps libgobuster.Set[netip.Addr] } // Error is the implementation of the error interface @@ -31,7 +32,7 @@ type GobusterDNS struct { globalopts *libgobuster.Options options *OptionsDNS isWildcard bool - wildcardIps libgobuster.StringSet + wildcardIps libgobuster.Set[netip.Addr] } func newCustomDialer(server string) func(ctx context.Context, network, address string) (net.Conn, error) { @@ -65,7 +66,7 @@ func NewGobusterDNS(globalopts *libgobuster.Options, opts *OptionsDNS) (*Gobuste g := GobusterDNS{ options: opts, globalopts: globalopts, - wildcardIps: libgobuster.NewStringSet(), + wildcardIps: libgobuster.NewSet[netip.Addr](), resolver: resolver, } return &g, nil @@ -76,11 +77,6 @@ func (d *GobusterDNS) Name() string { return "DNS enumeration" } -// RequestsPerRun returns the number of requests this plugin makes per single wordlist item -func (d *GobusterDNS) RequestsPerRun() int { - return 1 -} - // PreRun is the pre run implementation of gobusterdns func (d *GobusterDNS) PreRun(ctx context.Context) error { // Resolve a subdomain that probably shouldn't exist @@ -106,8 +102,8 @@ func (d *GobusterDNS) PreRun(ctx context.Context) error { return nil } -// Run is the process implementation of gobusterdns -func (d *GobusterDNS) Run(ctx context.Context, word string, resChannel chan<- libgobuster.Result) error { +// ProcessWord is the process implementation of gobusterdns +func (d *GobusterDNS) ProcessWord(ctx context.Context, word string, progress *libgobuster.Progress) error { subdomain := fmt.Sprintf("%s.%s", word, d.options.Domain) ips, err := d.dnsLookup(ctx, subdomain) if err == nil { @@ -126,10 +122,10 @@ func (d *GobusterDNS) Run(ctx context.Context, word string, resChannel chan<- li result.CNAME = cname } } - resChannel <- result + progress.ResultChan <- result } } else if d.globalopts.Verbose { - resChannel <- Result{ + progress.ResultChan <- Result{ Subdomain: subdomain, Found: false, ShowIPs: d.options.ShowIPs, @@ -139,6 +135,10 @@ func (d *GobusterDNS) Run(ctx context.Context, word string, resChannel chan<- li return nil } +func (d *GobusterDNS) AdditionalWords(word string) []string { + return []string{} +} + // GetConfigString returns the string representation of the current config func (d *GobusterDNS) GetConfigString() (string, error) { var buffer bytes.Buffer @@ -219,10 +219,10 @@ func (d *GobusterDNS) GetConfigString() (string, error) { return strings.TrimSpace(buffer.String()), nil } -func (d *GobusterDNS) dnsLookup(ctx context.Context, domain string) ([]string, error) { +func (d *GobusterDNS) dnsLookup(ctx context.Context, domain string) ([]netip.Addr, error) { ctx2, cancel := context.WithTimeout(ctx, d.options.Timeout) defer cancel() - return d.resolver.LookupHost(ctx2, domain) + return d.resolver.LookupNetIP(ctx2, "ip", domain) } func (d *GobusterDNS) dnsLookupCname(ctx context.Context, domain string) (string, error) { diff --git a/gobusterdns/result.go b/gobusterdns/result.go index d322f3d..b93501d 100644 --- a/gobusterdns/result.go +++ b/gobusterdns/result.go @@ -2,8 +2,15 @@ package gobusterdns import ( "bytes" - "fmt" + "net/netip" "strings" + + "github.com/fatih/color" +) + +var ( + yellow = color.New(color.FgYellow).FprintfFunc() + green = color.New(color.FgGreen).FprintfFunc() ) // Result represents a single result @@ -12,7 +19,7 @@ type Result struct { ShowCNAME bool Found bool Subdomain string - IPs []string + IPs []netip.Addr CNAME string } @@ -20,28 +27,25 @@ type Result struct { func (r Result) ResultToString() (string, error) { buf := &bytes.Buffer{} + c := green + if r.Found { - if _, err := fmt.Fprintf(buf, "Found: "); err != nil { - return "", err - } + c(buf, "Found: ") } else { - if _, err := fmt.Fprintf(buf, "Missed: "); err != nil { - return "", err - } + c = yellow + c(buf, "Missed: ") } if r.ShowIPs && r.Found { - if _, err := fmt.Fprintf(buf, "%s [%s]\n", r.Subdomain, strings.Join(r.IPs, ",")); err != nil { - return "", err + ips := make([]string, len(r.IPs)) + for i := range r.IPs { + ips[i] = r.IPs[i].String() } + c(buf, "%s [%s]\n", r.Subdomain, strings.Join(ips, ",")) } else if r.ShowCNAME && r.Found && r.CNAME != "" { - if _, err := fmt.Fprintf(buf, "%s [%s]\n", r.Subdomain, r.CNAME); err != nil { - return "", err - } + c(buf, "%s [%s]\n", r.Subdomain, r.CNAME) } else { - if _, err := fmt.Fprintf(buf, "%s\n", r.Subdomain); err != nil { - return "", err - } + c(buf, "%s\n", r.Subdomain) } s := buf.String() diff --git a/gobusterfuzz/gobusterfuzz.go b/gobusterfuzz/gobusterfuzz.go index 3b8cabf..b0dca52 100644 --- a/gobusterfuzz/gobusterfuzz.go +++ b/gobusterfuzz/gobusterfuzz.go @@ -5,6 +5,7 @@ import ( "bytes" "context" "fmt" + "net" "strings" "text/tabwriter" @@ -50,6 +51,8 @@ func NewGobusterFuzz(globalopts *libgobuster.Options, opts *OptionsFuzz) (*Gobus Timeout: opts.Timeout, UserAgent: opts.UserAgent, NoTLSValidation: opts.NoTLSValidation, + RetryOnTimeout: opts.RetryOnTimeout, + RetryAttempts: opts.RetryAttempts, } httpOpts := libgobuster.HTTPOptions{ @@ -75,24 +78,44 @@ func (d *GobusterFuzz) Name() string { return "fuzzing" } -// RequestsPerRun returns the number of requests this plugin makes per single wordlist item -func (d *GobusterFuzz) RequestsPerRun() int { - return 1 -} - // PreRun is the pre run implementation of gobusterfuzz func (d *GobusterFuzz) PreRun(ctx context.Context) error { return nil } -// Run is the process implementation of gobusterfuzz -func (d *GobusterFuzz) Run(ctx context.Context, word string, resChannel chan<- libgobuster.Result) error { - workingURL := strings.ReplaceAll(d.options.URL, "FUZZ", word) - statusCode, size, _, _, err := d.http.Request(ctx, workingURL, libgobuster.RequestOptions{}) - if err != nil { - return err +// ProcessWord is the process implementation of gobusterfuzz +func (d *GobusterFuzz) ProcessWord(ctx context.Context, word string, progress *libgobuster.Progress) error { + url := strings.ReplaceAll(d.options.URL, "FUZZ", word) + + tries := 1 + if d.options.RetryOnTimeout && d.options.RetryAttempts > 0 { + // add it so it will be the overall max requests + tries += d.options.RetryAttempts } - if statusCode != nil { + + var statusCode int + var size int64 + for i := 1; i <= tries; i++ { + var err error + statusCode, size, _, _, err = d.http.Request(ctx, url, libgobuster.RequestOptions{}) + if err != nil { + // check if it's a timeout and if we should try again and try again + // otherwise the timeout error is raised + if netErr, ok := err.(net.Error); ok && netErr.Timeout() && i != tries { + continue + } else if strings.Contains(err.Error(), "invalid control character in URL") { + // put error in error chan so it's printed out and ignore it + // so gobuster will not quit + progress.ErrorChan <- err + continue + } else { + return err + } + } + break + } + + if statusCode != 0 { resultStatus := true if helper.SliceContains(d.options.ExcludeLength, int(size)) { @@ -100,17 +123,17 @@ func (d *GobusterFuzz) Run(ctx context.Context, word string, resChannel chan<- l } if d.options.ExcludedStatusCodesParsed.Length() > 0 { - if d.options.ExcludedStatusCodesParsed.Contains(*statusCode) { + if d.options.ExcludedStatusCodesParsed.Contains(statusCode) { resultStatus = false } } if resultStatus || d.globalopts.Verbose { - resChannel <- Result{ + progress.ResultChan <- Result{ Verbose: d.globalopts.Verbose, Found: resultStatus, - Path: workingURL, - StatusCode: *statusCode, + Path: url, + StatusCode: statusCode, Size: size, } } @@ -118,6 +141,10 @@ func (d *GobusterFuzz) Run(ctx context.Context, word string, resChannel chan<- l return nil } +func (d *GobusterFuzz) AdditionalWords(word string) []string { + return []string{} +} + // GetConfigString returns the string representation of the current config func (d *GobusterFuzz) GetConfigString() (string, error) { var buffer bytes.Buffer diff --git a/gobusterfuzz/options.go b/gobusterfuzz/options.go index 5fc5b89..2ecb865 100644 --- a/gobusterfuzz/options.go +++ b/gobusterfuzz/options.go @@ -8,13 +8,13 @@ import ( type OptionsFuzz struct { libgobuster.HTTPOptions ExcludedStatusCodes string - ExcludedStatusCodesParsed libgobuster.IntSet + ExcludedStatusCodesParsed libgobuster.Set[int] ExcludeLength []int } // NewOptionsFuzz returns a new initialized OptionsFuzz func NewOptionsFuzz() *OptionsFuzz { return &OptionsFuzz{ - ExcludedStatusCodesParsed: libgobuster.NewIntSet(), + ExcludedStatusCodesParsed: libgobuster.NewSet[int](), } } diff --git a/gobusterfuzz/result.go b/gobusterfuzz/result.go index 0e3c8cc..b4e48f8 100644 --- a/gobusterfuzz/result.go +++ b/gobusterfuzz/result.go @@ -2,7 +2,13 @@ package gobusterfuzz import ( "bytes" - "fmt" + + "github.com/fatih/color" +) + +var ( + yellow = color.New(color.FgYellow).FprintfFunc() + green = color.New(color.FgGreen).FprintfFunc() ) // Result represents a single result @@ -18,30 +24,22 @@ type Result struct { func (r Result) ResultToString() (string, error) { buf := &bytes.Buffer{} + c := green + // Prefix if we're in verbose mode if r.Verbose { if r.Found { - if _, err := fmt.Fprintf(buf, "Found: "); err != nil { - return "", err - } + c(buf, "Found: ") } else { - if _, err := fmt.Fprintf(buf, "Missed: "); err != nil { - return "", err - } + c = yellow + c(buf, "Missed: ") } } else if r.Found { - if _, err := fmt.Fprintf(buf, "Found: "); err != nil { - return "", err - } + c(buf, "Found: ") } - if _, err := fmt.Fprintf(buf, "[Status=%d] [Length=%d] %s", r.StatusCode, r.Size, r.Path); err != nil { - return "", err - } - - if _, err := fmt.Fprintf(buf, "\n"); err != nil { - return "", err - } + c(buf, "[Status=%d] [Length=%d] %s", r.StatusCode, r.Size, r.Path) + c(buf, "\n") s := buf.String() return s, nil diff --git a/gobustergcs/gobustersgcs.go b/gobustergcs/gobustersgcs.go new file mode 100644 index 0000000..06e0aff --- /dev/null +++ b/gobustergcs/gobustersgcs.go @@ -0,0 +1,265 @@ +package gobustergcs + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "net" + "net/http" + "regexp" + "strings" + "text/tabwriter" + + "github.com/OJ/gobuster/v3/libgobuster" +) + +// GobusterGCS is the main type to implement the interface +type GobusterGCS struct { + options *OptionsGCS + globalopts *libgobuster.Options + http *libgobuster.HTTPClient + bucketRegex *regexp.Regexp +} + +// NewGobusterGCS creates a new initialized GobusterGCS +func NewGobusterGCS(globalopts *libgobuster.Options, opts *OptionsGCS) (*GobusterGCS, error) { + if globalopts == nil { + return nil, fmt.Errorf("please provide valid global options") + } + + if opts == nil { + return nil, fmt.Errorf("please provide valid plugin options") + } + + g := GobusterGCS{ + options: opts, + globalopts: globalopts, + } + + basicOptions := libgobuster.BasicHTTPOptions{ + Proxy: opts.Proxy, + Timeout: opts.Timeout, + UserAgent: opts.UserAgent, + NoTLSValidation: opts.NoTLSValidation, + RetryOnTimeout: opts.RetryOnTimeout, + RetryAttempts: opts.RetryAttempts, + } + + httpOpts := libgobuster.HTTPOptions{ + BasicHTTPOptions: basicOptions, + // needed so we can list bucket contents + FollowRedirect: true, + } + + h, err := libgobuster.NewHTTPClient(&httpOpts) + if err != nil { + return nil, err + } + g.http = h + // https://cloud.google.com/storage/docs/naming-buckets + g.bucketRegex = regexp.MustCompile(`^[a-z0-9][a-z0-9\-_.]{1,61}[a-z0-9](\.[a-z0-9][a-z0-9\-_.]{1,61}[a-z0-9])*$`) + + return &g, nil +} + +// Name should return the name of the plugin +func (s *GobusterGCS) Name() string { + return "GCS bucket enumeration" +} + +// PreRun is the pre run implementation of GobusterS3 +func (s *GobusterGCS) PreRun(ctx context.Context) error { + return nil +} + +// ProcessWord is the process implementation of GobusterS3 +func (s *GobusterGCS) ProcessWord(ctx context.Context, word string, progress *libgobuster.Progress) error { + // only check for valid bucket names + if !s.isValidBucketName(word) { + return nil + } + + bucketURL := fmt.Sprintf("https://storage.googleapis.com/storage/v1/b/%s/o?maxResults=%d", word, s.options.MaxFilesToList) + + tries := 1 + if s.options.RetryOnTimeout && s.options.RetryAttempts > 0 { + // add it so it will be the overall max requests + tries += s.options.RetryAttempts + } + + var statusCode int + var body []byte + for i := 1; i <= tries; i++ { + var err error + statusCode, _, _, body, err = s.http.Request(ctx, bucketURL, libgobuster.RequestOptions{ReturnBody: true}) + if err != nil { + // check if it's a timeout and if we should try again and try again + // otherwise the timeout error is raised + if netErr, ok := err.(net.Error); ok && netErr.Timeout() && i != tries { + continue + } else if strings.Contains(err.Error(), "invalid control character in URL") { + // put error in error chan so it's printed out and ignore it + // so gobuster will not quit + progress.ErrorChan <- err + continue + } else { + return err + } + } + break + } + + if statusCode == 0 || body == nil { + return nil + } + + // looks like 401, 403, and 404 are the only negative status codes + found := false + switch statusCode { + case http.StatusUnauthorized, + http.StatusForbidden, + http.StatusNotFound: + found = false + case http.StatusOK: + // listing enabled + found = true + default: + // default to found as we use negative status codes + found = true + } + + // nothing found, bail out + // may add the result later if we want to enable verbose output + if !found { + return nil + } + + extraStr := "" + if s.globalopts.Verbose { + // get status + var result map[string]interface{} + err := json.Unmarshal(body, &result) + + if err != nil { + return fmt.Errorf("could not parse response json: %w", err) + } + + if _, exist := result["error"]; exist { + // https://cloud.google.com/storage/docs/json_api/v1/status-codes + gcsError := GCSError{} + err := json.Unmarshal(body, &gcsError) + if err != nil { + return fmt.Errorf("could not parse error json: %w", err) + } + extraStr = fmt.Sprintf("Error: %s (%d)", gcsError.Error.Message, gcsError.Error.Code) + } else if v, exist := result["kind"]; exist && v == "storage#objects" { + // https://cloud.google.com/storage/docs/json_api/v1/status-codes + // bucket listing enabled + gcsListing := GCSListing{} + err := json.Unmarshal(body, &gcsListing) + if err != nil { + return fmt.Errorf("could not parse result json: %w", err) + } + extraStr = "Bucket Listing enabled: " + for _, x := range gcsListing.Items { + extraStr += fmt.Sprintf("%s (%sb), ", x.Name, x.Size) + } + extraStr = strings.TrimRight(extraStr, ", ") + } + } + + progress.ResultChan <- Result{ + Found: found, + BucketName: word, + Status: extraStr, + } + + return nil +} + +func (s *GobusterGCS) AdditionalWords(word string) []string { + return []string{} +} + +// GetConfigString returns the string representation of the current config +func (s *GobusterGCS) GetConfigString() (string, error) { + var buffer bytes.Buffer + bw := bufio.NewWriter(&buffer) + tw := tabwriter.NewWriter(bw, 0, 5, 3, ' ', 0) + o := s.options + + if _, err := fmt.Fprintf(tw, "[+] Threads:\t%d\n", s.globalopts.Threads); err != nil { + return "", err + } + + if s.globalopts.Delay > 0 { + if _, err := fmt.Fprintf(tw, "[+] Delay:\t%s\n", s.globalopts.Delay); err != nil { + return "", err + } + } + + wordlist := "stdin (pipe)" + if s.globalopts.Wordlist != "-" { + wordlist = s.globalopts.Wordlist + } + if _, err := fmt.Fprintf(tw, "[+] Wordlist:\t%s\n", wordlist); err != nil { + return "", err + } + + if s.globalopts.PatternFile != "" { + if _, err := fmt.Fprintf(tw, "[+] Patterns:\t%s (%d entries)\n", s.globalopts.PatternFile, len(s.globalopts.Patterns)); err != nil { + return "", err + } + } + + if o.Proxy != "" { + if _, err := fmt.Fprintf(tw, "[+] Proxy:\t%s\n", o.Proxy); err != nil { + return "", err + } + } + + if o.UserAgent != "" { + if _, err := fmt.Fprintf(tw, "[+] User Agent:\t%s\n", o.UserAgent); err != nil { + return "", err + } + } + + if _, err := fmt.Fprintf(tw, "[+] Timeout:\t%s\n", o.Timeout.String()); err != nil { + return "", err + } + + if s.globalopts.Verbose { + if _, err := fmt.Fprintf(tw, "[+] Verbose:\ttrue\n"); err != nil { + return "", err + } + } + + if _, err := fmt.Fprintf(tw, "[+] Maximum files to list:\t%d\n", o.MaxFilesToList); err != nil { + return "", err + } + + if err := tw.Flush(); err != nil { + return "", fmt.Errorf("error on tostring: %w", err) + } + + if err := bw.Flush(); err != nil { + return "", fmt.Errorf("error on tostring: %w", err) + } + + return strings.TrimSpace(buffer.String()), nil +} + +// https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html +func (s *GobusterGCS) isValidBucketName(bucketName string) bool { + if len(bucketName) > 222 || !s.bucketRegex.MatchString(bucketName) { + return false + } + if strings.HasPrefix(bucketName, "-") || strings.HasSuffix(bucketName, "-") || + strings.HasPrefix(bucketName, "_") || strings.HasSuffix(bucketName, "_") || + strings.HasPrefix(bucketName, ".") || strings.HasSuffix(bucketName, ".") { + return false + } + return true +} diff --git a/gobustergcs/options.go b/gobustergcs/options.go new file mode 100644 index 0000000..31a9d0f --- /dev/null +++ b/gobustergcs/options.go @@ -0,0 +1,16 @@ +package gobustergcs + +import ( + "github.com/OJ/gobuster/v3/libgobuster" +) + +// OptionsGCS is the struct to hold all options for this plugin +type OptionsGCS struct { + libgobuster.BasicHTTPOptions + MaxFilesToList int +} + +// NewOptionsGCS returns a new initialized OptionsS3 +func NewOptionsGCS() *OptionsGCS { + return &OptionsGCS{} +} diff --git a/gobustergcs/result.go b/gobustergcs/result.go new file mode 100644 index 0000000..55b124a --- /dev/null +++ b/gobustergcs/result.go @@ -0,0 +1,35 @@ +package gobustergcs + +import ( + "bytes" + + "github.com/fatih/color" +) + +var ( + green = color.New(color.FgGreen).FprintfFunc() +) + +// Result represents a single result +type Result struct { + Found bool + BucketName string + Status string +} + +// ResultToString converts the Result to it's textual representation +func (r Result) ResultToString() (string, error) { + buf := &bytes.Buffer{} + + c := green + + c(buf, "https://storage.googleapis.com/storage/v1/b/%s/o", r.BucketName) + + if r.Status != "" { + c(buf, " [%s]", r.Status) + } + c(buf, "\n") + + str := buf.String() + return str, nil +} diff --git a/gobustergcs/types.go b/gobustergcs/types.go new file mode 100644 index 0000000..72f8c3d --- /dev/null +++ b/gobustergcs/types.go @@ -0,0 +1,25 @@ +package gobustergcs + +// GCSError represents a returned error from GCS +type GCSError struct { + Error struct { + Code int `json:"code"` + Message string `json:"message"` + Errors []struct { + Message string `json:"message"` + Reason string `json:"reason"` + LocationType string `json:"locationType"` + Location string `json:"location"` + } `json:"errors"` + } `json:"error"` +} + +// GCSListing contains only a subset of returned properties +type GCSListing struct { + IsTruncated string `json:"nextPageToken"` + Items []struct { + Name string `json:"name"` + LastModified string `json:"updated"` + Size string `json:"size"` + } `json:"items"` +} diff --git a/gobusters3/gobusters3.go b/gobusters3/gobusters3.go index 7addb9e..bbfcdc4 100644 --- a/gobusters3/gobusters3.go +++ b/gobusters3/gobusters3.go @@ -6,6 +6,7 @@ import ( "context" "encoding/xml" "fmt" + "net" "net/http" "regexp" "strings" @@ -42,6 +43,8 @@ func NewGobusterS3(globalopts *libgobuster.Options, opts *OptionsS3) (*GobusterS Timeout: opts.Timeout, UserAgent: opts.UserAgent, NoTLSValidation: opts.NoTLSValidation, + RetryOnTimeout: opts.RetryOnTimeout, + RetryAttempts: opts.RetryAttempts, } httpOpts := libgobuster.HTTPOptions{ @@ -65,36 +68,55 @@ func (s *GobusterS3) Name() string { return "S3 bucket enumeration" } -// RequestsPerRun returns the number of requests this plugin makes per single wordlist item -func (s *GobusterS3) RequestsPerRun() int { - return 1 -} - // PreRun is the pre run implementation of GobusterS3 func (s *GobusterS3) PreRun(ctx context.Context) error { return nil } -// Run is the process implementation of GobusterS3 -func (s *GobusterS3) Run(ctx context.Context, word string, resChannel chan<- libgobuster.Result) error { +// ProcessWord is the process implementation of GobusterS3 +func (s *GobusterS3) ProcessWord(ctx context.Context, word string, progress *libgobuster.Progress) error { // only check for valid bucket names if !s.isValidBucketName(word) { return nil } bucketURL := fmt.Sprintf("https://%s.s3.amazonaws.com/?max-keys=%d", word, s.options.MaxFilesToList) - status, _, _, body, err := s.http.Request(ctx, bucketURL, libgobuster.RequestOptions{ReturnBody: true}) - if err != nil { - return err + + tries := 1 + if s.options.RetryOnTimeout && s.options.RetryAttempts > 0 { + // add it so it will be the overall max requests + tries += s.options.RetryAttempts } - if status == nil || body == nil { + var statusCode int + var body []byte + for i := 1; i <= tries; i++ { + var err error + statusCode, _, _, body, err = s.http.Request(ctx, bucketURL, libgobuster.RequestOptions{ReturnBody: true}) + if err != nil { + // check if it's a timeout and if we should try again and try again + // otherwise the timeout error is raised + if netErr, ok := err.(net.Error); ok && netErr.Timeout() && i != tries { + continue + } else if strings.Contains(err.Error(), "invalid control character in URL") { + // put error in error chan so it's printed out and ignore it + // so gobuster will not quit + progress.ErrorChan <- err + continue + } else { + return err + } + } + break + } + + if statusCode == 0 || body == nil { return nil } // looks like 404 and 400 are the only negative status codes found := false - switch *status { + switch statusCode { case http.StatusBadRequest: case http.StatusNotFound: found = false @@ -139,7 +161,7 @@ func (s *GobusterS3) Run(ctx context.Context, word string, resChannel chan<- lib } } - resChannel <- Result{ + progress.ResultChan <- Result{ Found: found, BucketName: word, Status: extraStr, @@ -148,6 +170,10 @@ func (s *GobusterS3) Run(ctx context.Context, word string, resChannel chan<- lib return nil } +func (d *GobusterS3) AdditionalWords(word string) []string { + return []string{} +} + // GetConfigString returns the string representation of the current config func (s *GobusterS3) GetConfigString() (string, error) { var buffer bytes.Buffer diff --git a/gobusters3/result.go b/gobusters3/result.go index 398ccc1..c2d1c62 100644 --- a/gobusters3/result.go +++ b/gobusters3/result.go @@ -2,7 +2,12 @@ package gobusters3 import ( "bytes" - "fmt" + + "github.com/fatih/color" +) + +var ( + green = color.New(color.FgGreen).FprintfFunc() ) // Result represents a single result @@ -16,19 +21,14 @@ type Result struct { func (r Result) ResultToString() (string, error) { buf := &bytes.Buffer{} - if _, err := fmt.Fprintf(buf, "http://%s.s3.amazonaws.com/", r.BucketName); err != nil { - return "", err - } + c := green + + c(buf, "http://%s.s3.amazonaws.com/", r.BucketName) if r.Status != "" { - if _, err := fmt.Fprintf(buf, " [%s]", r.Status); err != nil { - return "", err - } - } - - if _, err := fmt.Fprintf(buf, "\n"); err != nil { - return "", err + c(buf, " [%s]", r.Status) } + c(buf, "\n") str := buf.String() return str, nil diff --git a/gobustervhost/gobustervhost.go b/gobustervhost/gobustervhost.go index 2d43072..04bee58 100644 --- a/gobustervhost/gobustervhost.go +++ b/gobustervhost/gobustervhost.go @@ -5,6 +5,8 @@ import ( "bytes" "context" "fmt" + "net" + "net/http" "net/url" "strings" "text/tabwriter" @@ -16,12 +18,12 @@ import ( // GobusterVhost is the main type to implement the interface type GobusterVhost struct { - options *OptionsVhost - globalopts *libgobuster.Options - http *libgobuster.HTTPClient - domain string - baseline1 []byte - baseline2 []byte + options *OptionsVhost + globalopts *libgobuster.Options + http *libgobuster.HTTPClient + domain string + normalBody []byte + abnormalBody []byte } // NewGobusterVhost creates a new initialized GobusterDir @@ -44,6 +46,8 @@ func NewGobusterVhost(globalopts *libgobuster.Options, opts *OptionsVhost) (*Gob Timeout: opts.Timeout, UserAgent: opts.UserAgent, NoTLSValidation: opts.NoTLSValidation, + RetryOnTimeout: opts.RetryOnTimeout, + RetryAttempts: opts.RetryAttempts, } httpOpts := libgobuster.HTTPOptions{ @@ -69,11 +73,6 @@ func (v *GobusterVhost) Name() string { return "VHOST enumeration" } -// RequestsPerRun returns the number of requests this plugin makes per single wordlist item -func (v *GobusterVhost) RequestsPerRun() int { - return 1 -} - // PreRun is the pre run implementation of gobusterdir func (v *GobusterVhost) PreRun(ctx context.Context) error { // add trailing slash @@ -91,25 +90,25 @@ func (v *GobusterVhost) PreRun(ctx context.Context) error { v.domain = urlParsed.Host } - // request default vhost for baseline1 - _, _, _, tmp, err := v.http.Request(ctx, v.options.URL, libgobuster.RequestOptions{ReturnBody: true}) + // request default vhost for normalBody + _, _, _, body, err := v.http.Request(ctx, v.options.URL, libgobuster.RequestOptions{ReturnBody: true}) if err != nil { return fmt.Errorf("unable to connect to %s: %w", v.options.URL, err) } - v.baseline1 = tmp + v.normalBody = body - // request non existent vhost for baseline2 + // request non existent vhost for abnormalBody subdomain := fmt.Sprintf("%s.%s", uuid.New(), v.domain) - _, _, _, tmp, err = v.http.Request(ctx, v.options.URL, libgobuster.RequestOptions{Host: subdomain, ReturnBody: true}) + _, _, _, body, err = v.http.Request(ctx, v.options.URL, libgobuster.RequestOptions{Host: subdomain, ReturnBody: true}) if err != nil { return fmt.Errorf("unable to connect to %s: %w", v.options.URL, err) } - v.baseline2 = tmp + v.abnormalBody = body return nil } -// Run is the process implementation of gobusterdir -func (v *GobusterVhost) Run(ctx context.Context, word string, resChannel chan<- libgobuster.Result) error { +// ProcessWord is the process implementation of gobusterdir +func (v *GobusterVhost) ProcessWord(ctx context.Context, word string, progress *libgobuster.Progress) error { var subdomain string if v.options.AppendDomain { subdomain = fmt.Sprintf("%s.%s", word, v.domain) @@ -117,23 +116,49 @@ func (v *GobusterVhost) Run(ctx context.Context, word string, resChannel chan<- // wordlist needs to include full domains subdomain = word } - status, size, header, body, err := v.http.Request(ctx, v.options.URL, libgobuster.RequestOptions{Host: subdomain, ReturnBody: true}) - if err != nil { - return err + + tries := 1 + if v.options.RetryOnTimeout && v.options.RetryAttempts > 0 { + // add it so it will be the overall max requests + tries += v.options.RetryAttempts + } + + var statusCode int + var size int64 + var header http.Header + var body []byte + for i := 1; i <= tries; i++ { + var err error + statusCode, size, header, body, err = v.http.Request(ctx, v.options.URL, libgobuster.RequestOptions{Host: subdomain, ReturnBody: true}) + if err != nil { + // check if it's a timeout and if we should try again and try again + // otherwise the timeout error is raised + if netErr, ok := err.(net.Error); ok && netErr.Timeout() && i != tries { + continue + } else if strings.Contains(err.Error(), "invalid control character in URL") { + // put error in error chan so it's printed out and ignore it + // so gobuster will not quit + progress.ErrorChan <- err + continue + } else { + return err + } + } + break } // subdomain must not match default vhost and non existent vhost // or verbose mode is enabled - found := !bytes.Equal(body, v.baseline1) && !bytes.Equal(body, v.baseline2) + found := body != nil && !bytes.Equal(body, v.normalBody) && !bytes.Equal(body, v.abnormalBody) if (found && !helper.SliceContains(v.options.ExcludeLength, int(size))) || v.globalopts.Verbose { resultStatus := false if found { resultStatus = true } - resChannel <- Result{ + progress.ResultChan <- Result{ Found: resultStatus, Vhost: subdomain, - StatusCode: *status, + StatusCode: statusCode, Size: size, Header: header, } @@ -141,6 +166,10 @@ func (v *GobusterVhost) Run(ctx context.Context, word string, resChannel chan<- return nil } +func (v *GobusterVhost) AdditionalWords(word string) []string { + return []string{} +} + // GetConfigString returns the string representation of the current config func (v *GobusterVhost) GetConfigString() (string, error) { var buffer bytes.Buffer diff --git a/gobustervhost/result.go b/gobustervhost/result.go index e2c473a..1ae76d1 100644 --- a/gobustervhost/result.go +++ b/gobustervhost/result.go @@ -1,9 +1,19 @@ package gobustervhost import ( - "bytes" "fmt" "net/http" + + "github.com/fatih/color" +) + +var ( + white = color.New(color.FgWhite).SprintFunc() + yellow = color.New(color.FgYellow).SprintFunc() + green = color.New(color.FgGreen).SprintFunc() + blue = color.New(color.FgBlue).SprintFunc() + red = color.New(color.FgRed).SprintFunc() + cyan = color.New(color.FgCyan).SprintFunc() ) // Result represents a single result @@ -17,17 +27,29 @@ type Result struct { // ResultToString converts the Result to it's textual representation func (r Result) ResultToString() (string, error) { - buf := &bytes.Buffer{} - - statusText := "Missed" + statusText := yellow("Missed") if r.Found { - statusText = "Found" + statusText = green("Found") } - if _, err := fmt.Fprintf(buf, "%s: %s (Status: %d) [Size: %d]\n", statusText, r.Vhost, r.StatusCode, r.Size); err != nil { - return "", err + statusCodeColor := white + if r.StatusCode == 200 { + statusCodeColor = green + } else if r.StatusCode >= 300 && r.StatusCode < 400 { + statusCodeColor = cyan + } else if r.StatusCode >= 400 && r.StatusCode < 500 { + statusCodeColor = yellow + } else if r.StatusCode >= 500 && r.StatusCode < 600 { + statusCodeColor = red } - s := buf.String() - return s, nil + statusCode := statusCodeColor(fmt.Sprintf("Status: %d", r.StatusCode)) + + location := r.Header.Get("Location") + locationString := "" + if location != "" { + locationString = blue(fmt.Sprintf(" [--> %s]", location)) + } + + return fmt.Sprintf("%s: %s %s [Size: %d]%s\n", statusText, r.Vhost, statusCode, r.Size, locationString), nil } diff --git a/helper/helper.go b/helper/helper.go index 1efac08..e43ff35 100644 --- a/helper/helper.go +++ b/helper/helper.go @@ -9,12 +9,13 @@ import ( ) // ParseExtensions parses the extensions provided as a comma separated list -func ParseExtensions(extensions string) (libgobuster.StringSet, error) { +func ParseExtensions(extensions string) (libgobuster.Set[string], error) { + ret := libgobuster.NewSet[string]() + if extensions == "" { - return libgobuster.StringSet{}, nil + return ret, nil } - ret := libgobuster.NewStringSet() for _, e := range strings.Split(extensions, ",") { e = strings.TrimSpace(e) // remove leading . from extensions @@ -24,17 +25,18 @@ func ParseExtensions(extensions string) (libgobuster.StringSet, error) { } // ParseCommaSeparatedInt parses the status codes provided as a comma separated list -func ParseCommaSeparatedInt(inputString string) (libgobuster.IntSet, error) { +func ParseCommaSeparatedInt(inputString string) (libgobuster.Set[int], error) { + ret := libgobuster.NewSet[int]() + if inputString == "" { - return libgobuster.IntSet{}, nil + return ret, nil } - ret := libgobuster.NewIntSet() for _, c := range strings.Split(inputString, ",") { c = strings.TrimSpace(c) i, err := strconv.Atoi(c) if err != nil { - return libgobuster.IntSet{}, fmt.Errorf("invalid string given: %s", c) + return libgobuster.NewSet[int](), fmt.Errorf("invalid string given: %s", c) } ret.Add(i) } diff --git a/helper/helper_test.go b/helper/helper_test.go index 3ab9bfd..8d0d784 100644 --- a/helper/helper_test.go +++ b/helper/helper_test.go @@ -12,14 +12,14 @@ func TestParseExtensions(t *testing.T) { var tt = []struct { testName string extensions string - expectedExtensions libgobuster.StringSet + expectedExtensions libgobuster.Set[string] expectedError string }{ - {"Valid extensions", "php,asp,txt", libgobuster.StringSet{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, - {"Spaces", "php, asp , txt", libgobuster.StringSet{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, - {"Double extensions", "php,asp,txt,php,asp,txt", libgobuster.StringSet{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, - {"Leading dot", ".php,asp,.txt", libgobuster.StringSet{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, - {"Empty string", "", libgobuster.NewStringSet(), "invalid extension string provided"}, + {"Valid extensions", "php,asp,txt", libgobuster.Set[string]{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, + {"Spaces", "php, asp , txt", libgobuster.Set[string]{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, + {"Double extensions", "php,asp,txt,php,asp,txt", libgobuster.Set[string]{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, + {"Leading dot", ".php,asp,.txt", libgobuster.Set[string]{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, + {"Empty string", "", libgobuster.NewSet[string](), "invalid extension string provided"}, } for _, x := range tt { @@ -43,15 +43,15 @@ func TestParseCommaSeparatedInt(t *testing.T) { var tt = []struct { testName string stringCodes string - expectedCodes libgobuster.IntSet + expectedCodes libgobuster.Set[int] expectedError string }{ - {"Valid codes", "200,100,202", libgobuster.IntSet{Set: map[int]bool{100: true, 200: true, 202: true}}, ""}, - {"Spaces", "200, 100 , 202", libgobuster.IntSet{Set: map[int]bool{100: true, 200: true, 202: true}}, ""}, - {"Double codes", "200, 100, 202, 100", libgobuster.IntSet{Set: map[int]bool{100: true, 200: true, 202: true}}, ""}, - {"Invalid code", "200,AAA", libgobuster.NewIntSet(), "invalid string given: AAA"}, - {"Invalid integer", "2000000000000000000000000000000", libgobuster.NewIntSet(), "invalid string given: 2000000000000000000000000000000"}, - {"Empty string", "", libgobuster.NewIntSet(), "invalid string provided"}, + {"Valid codes", "200,100,202", libgobuster.Set[int]{Set: map[int]bool{100: true, 200: true, 202: true}}, ""}, + {"Spaces", "200, 100 , 202", libgobuster.Set[int]{Set: map[int]bool{100: true, 200: true, 202: true}}, ""}, + {"Double codes", "200, 100, 202, 100", libgobuster.Set[int]{Set: map[int]bool{100: true, 200: true, 202: true}}, ""}, + {"Invalid code", "200,AAA", libgobuster.NewSet[int](), "invalid string given: AAA"}, + {"Invalid integer", "2000000000000000000000000000000", libgobuster.NewSet[int](), "invalid string given: 2000000000000000000000000000000"}, + {"Empty string", "", libgobuster.NewSet[int](), "invalid string provided"}, } for _, x := range tt { @@ -74,14 +74,14 @@ func BenchmarkParseExtensions(b *testing.B) { var tt = []struct { testName string extensions string - expectedExtensions libgobuster.StringSet + expectedExtensions libgobuster.Set[string] expectedError string }{ - {"Valid extensions", "php,asp,txt", libgobuster.StringSet{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, - {"Spaces", "php, asp , txt", libgobuster.StringSet{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, - {"Double extensions", "php,asp,txt,php,asp,txt", libgobuster.StringSet{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, - {"Leading dot", ".php,asp,.txt", libgobuster.StringSet{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, - {"Empty string", "", libgobuster.NewStringSet(), "invalid extension string provided"}, + {"Valid extensions", "php,asp,txt", libgobuster.Set[string]{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, + {"Spaces", "php, asp , txt", libgobuster.Set[string]{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, + {"Double extensions", "php,asp,txt,php,asp,txt", libgobuster.Set[string]{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, + {"Leading dot", ".php,asp,.txt", libgobuster.Set[string]{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, + {"Empty string", "", libgobuster.NewSet[string](), "invalid extension string provided"}, } for _, x := range tt { @@ -98,15 +98,15 @@ func BenchmarkParseCommaSeparatedInt(b *testing.B) { var tt = []struct { testName string stringCodes string - expectedCodes libgobuster.IntSet + expectedCodes libgobuster.Set[int] expectedError string }{ - {"Valid codes", "200,100,202", libgobuster.IntSet{Set: map[int]bool{100: true, 200: true, 202: true}}, ""}, - {"Spaces", "200, 100 , 202", libgobuster.IntSet{Set: map[int]bool{100: true, 200: true, 202: true}}, ""}, - {"Double codes", "200, 100, 202, 100", libgobuster.IntSet{Set: map[int]bool{100: true, 200: true, 202: true}}, ""}, - {"Invalid code", "200,AAA", libgobuster.NewIntSet(), "invalid string given: AAA"}, - {"Invalid integer", "2000000000000000000000000000000", libgobuster.NewIntSet(), "invalid string given: 2000000000000000000000000000000"}, - {"Empty string", "", libgobuster.NewIntSet(), "invalid string string provided"}, + {"Valid codes", "200,100,202", libgobuster.Set[int]{Set: map[int]bool{100: true, 200: true, 202: true}}, ""}, + {"Spaces", "200, 100 , 202", libgobuster.Set[int]{Set: map[int]bool{100: true, 200: true, 202: true}}, ""}, + {"Double codes", "200, 100, 202, 100", libgobuster.Set[int]{Set: map[int]bool{100: true, 200: true, 202: true}}, ""}, + {"Invalid code", "200,AAA", libgobuster.NewSet[int](), "invalid string given: AAA"}, + {"Invalid integer", "2000000000000000000000000000000", libgobuster.NewSet[int](), "invalid string given: 2000000000000000000000000000000"}, + {"Empty string", "", libgobuster.NewSet[int](), "invalid string string provided"}, } for _, x := range tt { diff --git a/libgobuster/helpers.go b/libgobuster/helpers.go index d8ec090..fbc054a 100644 --- a/libgobuster/helpers.go +++ b/libgobuster/helpers.go @@ -5,47 +5,41 @@ import ( "errors" "fmt" "io" - "sort" "strings" ) -// IntSet is a set of Ints -type IntSet struct { - Set map[int]bool +// Set is a set of Ts +type Set[T comparable] struct { + Set map[T]bool } -// StringSet is a set of Strings -type StringSet struct { - Set map[string]bool -} - -// NewStringSet creates a new initialized StringSet -func NewStringSet() StringSet { - return StringSet{Set: map[string]bool{}} +// NewSSet creates a new initialized Set +func NewSet[T comparable]() Set[T] { + return Set[T]{Set: map[T]bool{}} } // Add an element to a set -func (set *StringSet) Add(s string) bool { +func (set *Set[T]) Add(s T) bool { _, found := set.Set[s] set.Set[s] = true return !found } // AddRange adds a list of elements to a set -func (set *StringSet) AddRange(ss []string) { +func (set *Set[T]) AddRange(ss []T) { for _, s := range ss { set.Set[s] = true } } // Contains tests if an element is in a set -func (set *StringSet) Contains(s string) bool { +func (set *Set[T]) Contains(s T) bool { _, found := set.Set[s] return found } // ContainsAny checks if any of the elements exist -func (set *StringSet) ContainsAny(ss []string) bool { +func (set *Set[T]) ContainsAny(ss []T) bool { for _, s := range ss { if set.Set[s] { return true @@ -55,58 +49,21 @@ func (set *StringSet) ContainsAny(ss []string) bool { } // Length returns the length of the Set -func (set *StringSet) Length() int { +func (set *Set[T]) Length() int { return len(set.Set) } // Stringify the set -func (set *StringSet) Stringify() string { +func (set *Set[T]) Stringify() string { values := make([]string, len(set.Set)) i := 0 for s := range set.Set { - values[i] = s + values[i] = fmt.Sprint(s) i++ } return strings.Join(values, ",") } -// NewIntSet creates a new initialized IntSet -func NewIntSet() IntSet { - return IntSet{Set: map[int]bool{}} -} - -// Add adds an element to a set -func (set *IntSet) Add(i int) bool { - _, found := set.Set[i] - set.Set[i] = true - return !found -} - -// Contains tests if an element is in a set -func (set *IntSet) Contains(i int) bool { - _, found := set.Set[i] - return found -} - -// Stringify the set -func (set *IntSet) Stringify() string { - values := make([]int, len(set.Set)) - i := 0 - for s := range set.Set { - values[i] = s - i++ - } - sort.Ints(values) - - delim := "," - return strings.Trim(strings.Join(strings.Fields(fmt.Sprint(values)), delim), "[]") -} - -// Length returns the length of the Set -func (set *IntSet) Length() int { - return len(set.Set) -} - func lineCounter(r io.Reader) (int, error) { buf := make([]byte, 32*1024) count := 1 diff --git a/libgobuster/helpers_test.go b/libgobuster/helpers_test.go index 9c5d057..900543b 100644 --- a/libgobuster/helpers_test.go +++ b/libgobuster/helpers_test.go @@ -2,77 +2,109 @@ package libgobuster import ( "errors" + "fmt" "strings" "testing" "testing/iotest" ) -func TestNewStringSet(t *testing.T) { +func TestNewSet(t *testing.T) { t.Parallel() - if NewStringSet().Set == nil { - t.Fatal("newStringSet returned nil Set") + if NewSet[string]().Set == nil { + t.Fatal("NewSet[string] returned nil Set") + } + + if NewSet[int]().Set == nil { + t.Fatal("NewSet[int] returned nil Set") } } -func TestNewIntSet(t *testing.T) { +func TestSetAdd(t *testing.T) { t.Parallel() - if NewIntSet().Set == nil { - t.Fatal("newIntSet returned nil Set") - } -} - -func TestStringSetAdd(t *testing.T) { - t.Parallel() - x := NewStringSet() + x := NewSet[string]() x.Add("test") if len(x.Set) != 1 { - t.Fatalf("Unexpected size. Should have 1 Got %v", len(x.Set)) + t.Fatalf("Unexpected string size. Should have 1 Got %v", len(x.Set)) + } + + y := NewSet[int]() + y.Add(1) + if len(y.Set) != 1 { + t.Fatalf("Unexpected int size. Should have 1 Got %v", len(y.Set)) } } -func TestStringSetAddDouble(t *testing.T) { +func TestSetAddDouble(t *testing.T) { t.Parallel() - x := NewStringSet() + x := NewSet[string]() x.Add("test") x.Add("test") if len(x.Set) != 1 { - t.Fatalf("Unexpected size. Should have 1 Got %d", len(x.Set)) + t.Fatalf("Unexpected string size. Should be 1 (unique) Got %v", len(x.Set)) + } + + y := NewSet[int]() + y.Add(1) + y.Add(1) + if len(y.Set) != 1 { + t.Fatalf("Unexpected int size. Should be 1 (unique) Got %v", len(y.Set)) } } -func TestStringSetAddRange(t *testing.T) { +func TestSetAddRange(t *testing.T) { t.Parallel() - x := NewStringSet() + x := NewSet[string]() x.AddRange([]string{"string1", "string2"}) if len(x.Set) != 2 { - t.Fatalf("Unexpected size. Should have 2 Got %d", len(x.Set)) + t.Fatalf("Unexpected string size. Should have 2 Got %v", len(x.Set)) + } + + y := NewSet[int]() + y.AddRange([]int{1, 2}) + if len(y.Set) != 2 { + t.Fatalf("Unexpected int size. Should have 2 Got %v", len(y.Set)) } } -func TestStringSetAddRangeDouble(t *testing.T) { +func TestSetAddRangeDouble(t *testing.T) { t.Parallel() - x := NewStringSet() + x := NewSet[string]() x.AddRange([]string{"string1", "string2", "string1", "string2"}) if len(x.Set) != 2 { - t.Fatalf("Unexpected size. Should have 2 Got %d", len(x.Set)) + t.Fatalf("Unexpected string size. Should be 2 (unique) Got %v", len(x.Set)) + } + + y := NewSet[int]() + y.AddRange([]int{1, 2, 1, 2}) + if len(y.Set) != 2 { + t.Fatalf("Unexpected int size. Should be 2 (unique) Got %v", len(y.Set)) } } -func TestStringSetContains(t *testing.T) { +func TestSetContains(t *testing.T) { t.Parallel() - x := NewStringSet() + x := NewSet[string]() v := []string{"string1", "string2", "1234", "5678"} x.AddRange(v) - for _, y := range v { - if !x.Contains(y) { - t.Fatalf("Did not find value %s in array. %v", y, x.Set) + for _, i := range v { + if !x.Contains(i) { + t.Fatalf("Did not find value %s in array. %v", i, x.Set) + } + } + + y := NewSet[int]() + v2 := []int{1, 2312, 123121, 999, -99} + y.AddRange(v2) + for _, i := range v2 { + if !y.Contains(i) { + t.Fatalf("Did not find value %d in array. %v", i, y.Set) } } } -func TestStringSetContainsAny(t *testing.T) { +func TestSetContainsAny(t *testing.T) { t.Parallel() - x := NewStringSet() + x := NewSet[string]() v := []string{"string1", "string2", "1234", "5678"} x.AddRange(v) if !x.ContainsAny(v) { @@ -83,70 +115,45 @@ func TestStringSetContainsAny(t *testing.T) { if x.ContainsAny([]string{"mmmm", "nnnnn"}) { t.Fatal("Found unexpected values") } + + y := NewSet[int]() + v2 := []int{1, 2312, 123121, 999, -99} + y.AddRange(v2) + if !y.ContainsAny(v2) { + t.Fatalf("Did not find any") + } + + // test not found + if y.ContainsAny([]int{9235, 2398532}) { + t.Fatal("Found unexpected values") + } } -func TestStringSetStringify(t *testing.T) { +func TestSetStringify(t *testing.T) { t.Parallel() - x := NewStringSet() + x := NewSet[string]() v := []string{"string1", "string2", "1234", "5678"} x.AddRange(v) z := x.Stringify() // order is random - for _, y := range v { - if !strings.Contains(z, y) { - t.Fatalf("Did not find value %q in %q", y, z) + for _, i := range v { + if !strings.Contains(z, i) { + t.Fatalf("Did not find value %q in %q", i, z) } } -} -func TestIntSetAdd(t *testing.T) { - t.Parallel() - x := NewIntSet() - x.Add(1) - if len(x.Set) != 1 { - t.Fatalf("Unexpected size. Should have 1 Got %d", len(x.Set)) - } -} - -func TestIntSetAddDouble(t *testing.T) { - t.Parallel() - x := NewIntSet() - x.Add(1) - x.Add(1) - if len(x.Set) != 1 { - t.Fatalf("Unexpected size. Should have 1 Got %d", len(x.Set)) - } -} - -func TestIntSetContains(t *testing.T) { - t.Parallel() - x := NewIntSet() - v := []int{1, 2, 3, 4} - for _, y := range v { - x.Add(y) - } - for _, y := range v { - if !x.Contains(y) { - t.Fatalf("Did not find value %d in array. %v", y, x.Set) + y := NewSet[int]() + v2 := []int{1, 2312, 123121, 999, -99} + y.AddRange(v2) + z = y.Stringify() + // order is random + for _, i := range v2 { + if !strings.Contains(z, fmt.Sprint(i)) { + t.Fatalf("Did not find value %q in %q", i, z) } } } -func TestIntSetStringify(t *testing.T) { - t.Parallel() - x := NewIntSet() - v := []int{1, 3, 2, 4} - expected := "1,2,3,4" - for _, y := range v { - x.Add(y) - } - z := x.Stringify() - // should be sorted - if expected != z { - t.Fatalf("Expected %q got %q", expected, z) - } -} - func TestLineCounter(t *testing.T) { t.Parallel() var tt = []struct { diff --git a/libgobuster/http.go b/libgobuster/http.go index 38e11aa..8ea791d 100644 --- a/libgobuster/http.go +++ b/libgobuster/http.go @@ -97,14 +97,14 @@ func NewHTTPClient(opt *HTTPOptions) (*HTTPClient, error) { // Request makes an http request and returns the status, the content length, the headers, the body and an error // if you want the body returned set the corresponding property inside RequestOptions -func (client *HTTPClient) Request(ctx context.Context, fullURL string, opts RequestOptions) (*int, int64, http.Header, []byte, error) { +func (client *HTTPClient) Request(ctx context.Context, fullURL string, opts RequestOptions) (int, int64, http.Header, []byte, error) { resp, err := client.makeRequest(ctx, fullURL, opts.Host, opts.Body) if err != nil { // ignore context canceled errors if errors.Is(ctx.Err(), context.Canceled) { - return nil, 0, nil, nil, nil + return 0, 0, nil, nil, nil } - return nil, 0, nil, nil, err + return 0, 0, nil, nil, err } defer resp.Body.Close() @@ -113,7 +113,7 @@ func (client *HTTPClient) Request(ctx context.Context, fullURL string, opts Requ if opts.ReturnBody { body, err = io.ReadAll(resp.Body) if err != nil { - return nil, 0, nil, nil, fmt.Errorf("could not read body %w", err) + return 0, 0, nil, nil, fmt.Errorf("could not read body %w", err) } length = int64(len(body)) } else { @@ -121,11 +121,11 @@ func (client *HTTPClient) Request(ctx context.Context, fullURL string, opts Requ // absolutely needed so golang will reuse connections! length, err = io.Copy(io.Discard, resp.Body) if err != nil { - return nil, 0, nil, nil, err + return 0, 0, nil, nil, err } } - return &resp.StatusCode, length, resp.Header, body, nil + return resp.StatusCode, length, resp.Header, body, nil } func (client *HTTPClient) makeRequest(ctx context.Context, fullURL, host string, data io.Reader) (*http.Response, error) { diff --git a/libgobuster/http_test.go b/libgobuster/http_test.go index 1602be2..3cc3612 100644 --- a/libgobuster/http_test.go +++ b/libgobuster/http_test.go @@ -59,7 +59,7 @@ func TestRequest(t *testing.T) { if err != nil { t.Fatalf("Got Error: %v", err) } - if *status != 200 { + if status != 200 { t.Fatalf("Invalid status returned: %d", status) } if length != int64(len(ret)) { diff --git a/libgobuster/interfaces.go b/libgobuster/interfaces.go index 3e0e5f6..28d7e2b 100644 --- a/libgobuster/interfaces.go +++ b/libgobuster/interfaces.go @@ -5,9 +5,9 @@ import "context" // GobusterPlugin is an interface which plugins must implement type GobusterPlugin interface { Name() string - RequestsPerRun() int PreRun(context.Context) error - Run(context.Context, string, chan<- Result) error + ProcessWord(context.Context, string, *Progress) error + AdditionalWords(string) []string GetConfigString() (string, error) } diff --git a/libgobuster/libgobuster.go b/libgobuster/libgobuster.go index 67b5fef..ad4bedd 100644 --- a/libgobuster/libgobuster.go +++ b/libgobuster/libgobuster.go @@ -9,6 +9,8 @@ import ( "strings" "sync" "time" + + "github.com/fatih/color" ) // PATTERN is the pattern for wordlist replacements in pattern file @@ -25,15 +27,11 @@ type ResultToStringFunc func(*Gobuster, *Result) (*string, error) // Gobuster is the main object when creating a new run type Gobuster struct { - Opts *Options - RequestsExpected int - RequestsIssued int - RequestsCountMutex *sync.RWMutex - plugin GobusterPlugin - resultChan chan Result - errorChan chan error - LogInfo *log.Logger - LogError *log.Logger + Opts *Options + plugin GobusterPlugin + LogInfo *log.Logger + LogError *log.Logger + Progress *Progress } // NewGobuster returns a new Gobuster object @@ -41,31 +39,13 @@ func NewGobuster(opts *Options, plugin GobusterPlugin) (*Gobuster, error) { var g Gobuster g.Opts = opts g.plugin = plugin - g.RequestsCountMutex = new(sync.RWMutex) - g.resultChan = make(chan Result) - g.errorChan = make(chan error) g.LogInfo = log.New(os.Stdout, "", log.LstdFlags) - g.LogError = log.New(os.Stderr, "[ERROR] ", log.LstdFlags) + g.LogError = log.New(os.Stderr, color.New(color.FgRed).Sprint("[ERROR] "), log.LstdFlags) + g.Progress = NewProgress() return &g, nil } -// Results returns a channel of Results -func (g *Gobuster) Results() <-chan Result { - return g.resultChan -} - -// Errors returns a channel of errors -func (g *Gobuster) Errors() <-chan error { - return g.errorChan -} - -func (g *Gobuster) incrementRequests() { - g.RequestsCountMutex.Lock() - g.RequestsIssued += g.plugin.RequestsPerRun() - g.RequestsCountMutex.Unlock() -} - func (g *Gobuster) worker(ctx context.Context, wordChan <-chan string, wg *sync.WaitGroup) { defer wg.Done() for { @@ -77,7 +57,7 @@ func (g *Gobuster) worker(ctx context.Context, wordChan <-chan string, wg *sync. if !ok { return } - g.incrementRequests() + g.Progress.incrementRequests() wordCleaned := strings.TrimSpace(word) // Skip "comment" (starts with #), as well as empty lines @@ -86,10 +66,10 @@ func (g *Gobuster) worker(ctx context.Context, wordChan <-chan string, wg *sync. } // Mode-specific processing - err := g.plugin.Run(ctx, wordCleaned, g.resultChan) + err := g.plugin.ProcessWord(ctx, wordCleaned, g.Progress) if err != nil { // do not exit and continue - g.errorChan <- err + g.Progress.ErrorChan <- err continue } @@ -117,15 +97,17 @@ func (g *Gobuster) getWordlist() (*bufio.Scanner, error) { return nil, fmt.Errorf("failed to get number of lines: %w", err) } - g.RequestsIssued = 0 - // calcutate expected requests - g.RequestsExpected = lines - if g.Opts.PatternFile != "" { - g.RequestsExpected += lines * len(g.Opts.Patterns) - } + g.Progress.IncrementTotalRequests(lines) - g.RequestsExpected *= g.plugin.RequestsPerRun() + // call the function once with a dummy entry to receive the number + // of custom words per wordlist word + customWordsLen := len(g.plugin.AdditionalWords("dummy")) + if customWordsLen > 0 { + origExpected := g.Progress.RequestsExpected() + inc := origExpected * customWordsLen + g.Progress.IncrementTotalRequests(inc) + } // rewind wordlist _, err = wordlist.Seek(0, 0) @@ -138,8 +120,8 @@ func (g *Gobuster) getWordlist() (*bufio.Scanner, error) { // Run the busting of the website with the given // set of settings from the command line. func (g *Gobuster) Run(ctx context.Context) error { - defer close(g.resultChan) - defer close(g.errorChan) + defer close(g.Progress.ResultChan) + defer close(g.Progress.ErrorChan) if err := g.plugin.PreRun(ctx); err != nil { return err @@ -180,6 +162,15 @@ Scan: case wordChan <- w: } } + + for _, w := range g.plugin.AdditionalWords(word) { + select { + // need to check here too otherwise wordChan will block + case <-ctx.Done(): + break Scan + case wordChan <- w: + } + } } } close(wordChan) diff --git a/libgobuster/options_http.go b/libgobuster/options_http.go index 4748244..359f552 100644 --- a/libgobuster/options_http.go +++ b/libgobuster/options_http.go @@ -10,6 +10,8 @@ type BasicHTTPOptions struct { Proxy string NoTLSValidation bool Timeout time.Duration + RetryOnTimeout bool + RetryAttempts int } // HTTPOptions is the struct to pass in all http options to Gobuster diff --git a/libgobuster/progress.go b/libgobuster/progress.go new file mode 100644 index 0000000..64a3182 --- /dev/null +++ b/libgobuster/progress.go @@ -0,0 +1,46 @@ +package libgobuster + +import "sync" + +type Progress struct { + requestsExpectedMutex *sync.RWMutex + requestsExpected int + requestsCountMutex *sync.RWMutex + requestsIssued int + ResultChan chan Result + ErrorChan chan error +} + +func NewProgress() *Progress { + var p Progress + p.requestsIssued = 0 + p.requestsExpectedMutex = new(sync.RWMutex) + p.requestsCountMutex = new(sync.RWMutex) + p.ResultChan = make(chan Result) + p.ErrorChan = make(chan error) + return &p +} + +func (p *Progress) RequestsExpected() int { + p.requestsExpectedMutex.RLock() + defer p.requestsExpectedMutex.RUnlock() + return p.requestsExpected +} + +func (p *Progress) RequestsIssued() int { + p.requestsCountMutex.RLock() + defer p.requestsCountMutex.RUnlock() + return p.requestsIssued +} + +func (p *Progress) incrementRequests() { + p.requestsCountMutex.Lock() + defer p.requestsCountMutex.Unlock() + p.requestsIssued++ +} + +func (p *Progress) IncrementTotalRequests(by int) { + p.requestsCountMutex.Lock() + defer p.requestsCountMutex.Unlock() + p.requestsExpected += by +} diff --git a/libgobuster/version.go b/libgobuster/version.go index a19dbd1..8ce0435 100644 --- a/libgobuster/version.go +++ b/libgobuster/version.go @@ -2,5 +2,5 @@ package libgobuster const ( // VERSION contains the current gobuster version - VERSION = "3.1.0" + VERSION = "3.2.0-dev" ) diff --git a/make.bat b/make.bat deleted file mode 100644 index 6149768..0000000 --- a/make.bat +++ /dev/null @@ -1,150 +0,0 @@ -@echo off - -SET ARG=%1 -SET TARGET=.\build -SET BUILDARGS=-ldflags="-s -w" -trimpath - -IF "%ARG%"=="test" ( - CALL :Test - GOTO Done -) - -IF "%ARG%"=="clean" ( - del /F /Q %TARGET%\*.* - go clean ./... - echo Done. - GOTO Done -) - -IF "%ARG%"=="windows" ( - CALL :Windows - GOTO Done -) - -IF "%ARG%"=="darwin" ( - CALL :Darwin - GOTO Done -) - -IF "%ARG%"=="linux" ( - CALL :Linux - GOTO Done -) - -IF "%ARG%"=="update" ( - CALL :Update - GOTO Done -) - -IF "%ARG%"=="fmt" ( - CALL :Fmt - GOTO Done -) - -IF "%ARG%"=="lint" ( - CALL :Lint - GOTO Done -) - -IF "%ARG%"=="all" ( - CALL :Fmt - CALL :Update - CALL :Lint - CALL :Test - CALL :Darwin - CALL :Linux - CALL :Windows - GOTO Done -) - -IF "%ARG%"=="" ( - go build -o .\gobuster.exe - GOTO Done -) - -GOTO Done - -:Test -set GO111MODULE=on -set CGO_ENABLED=0 -echo Testing ... -go test -v ./... -echo Done -EXIT /B 0 - -:Lint -set GO111MODULE=on -echo Linting ... -go get -u github.com/golangci/golangci-lint@master -golangci-lint run ./... -rem remove test deps -go mod tidy -echo Done - -:Fmt -set GO111MODULE=on -echo Formatting ... -go fmt ./... -echo Done. -EXIT /B 0 - -:Update -set GO111MODULE=on -echo Updating ... -go get -u -go mod tidy -v -echo Done. -EXIT /B 0 - -:Darwin -set GOOS=darwin -set GOARCH=amd64 -set GO111MODULE=on -set CGO_ENABLED=0 -echo Building for %GOOS% %GOARCH% ... -set DIR=%TARGET%\gobuster-%GOOS%-%GOARCH% -mkdir %DIR% 2> NUL -go build %BUILDARGS% -o %DIR%\gobuster -set GOARCH=386 -echo Building for %GOOS% %GOARCH% ... -set DIR=%TARGET%\gobuster-%GOOS%-%GOARCH% -mkdir %DIR% 2> NUL -go build %BUILDARGS% -o %DIR%\gobuster -echo Done. -EXIT /B 0 - -:Linux -set GOOS=linux -set GOARCH=amd64 -set GO111MODULE=on -set CGO_ENABLED=0 -echo Building for %GOOS% %GOARCH% ... -set DIR=%TARGET%\gobuster-%GOOS%-%GOARCH% -mkdir %DIR% 2> NUL -go build %BUILDARGS% -o %DIR%\gobuster -set GOARCH=386 -echo Building for %GOOS% %GOARCH% ... -set DIR=%TARGET%\gobuster-%GOOS%-%GOARCH% -mkdir %DIR% 2> NUL -go build %BUILDARGS% -o %DIR%\gobuster -echo Done. -EXIT /B 0 - -:Windows -set GOOS=windows -set GOARCH=amd64 -set GO111MODULE=on -set CGO_ENABLED=0 -echo Building for %GOOS% %GOARCH% ... -set DIR=%TARGET%\gobuster-%GOOS%-%GOARCH% -mkdir %DIR% 2> NUL -go build %BUILDARGS% -o %DIR%\gobuster.exe -set GOARCH=386 -echo Building for %GOOS% %GOARCH% ... -set DIR=%TARGET%\gobuster-%GOOS%-%GOARCH% -mkdir %DIR% 2> NUL -go build %BUILDARGS% -o %DIR%\gobuster.exe -echo Done. -EXIT /B 0 - -:Done