Merge pull request #59 from projectdiscovery/dev

v0.0.5 Release

Author: ehsandeep

README.md

@@ -34,6 +34,7 @@ A fast and configurable TLS grabber focused on TLS based **data collection and a
  - **Auto TLS Fallback** for older TLS version
  - **Pre Handshake** TLS connection (early termination)
  - Customizable **Cipher / SNI / TLS** selection
+ - **JARM/JA3** TLS Fingerprint
  - **TLS Misconfigurations**
  - **HOST, IP, URL** and **CIDR** input
  - STD **IN/OUT** and **TXT/JSON** output
@@ -77,6 +78,7 @@ PROBES:
    -cipher              display used cipher
    -hash string         display certificate fingerprint hashes (md5,sha1,sha256)
    -jarm                display jarm fingerprint hash
+   -ja3                 display ja3 fingerprint hash (using ztls)
    -tps, -probe-status  display tls probe status
 
 MISCONFIGURATIONS:
@@ -89,15 +91,18 @@ CONFIGURATIONS:
    -r, -resolvers string[]      list of resolvers to use
    -cc, -cacert string          client certificate authority file
    -ci, -cipher-input string[]  ciphers to use with tls connection
-   -sni string                  tls sni hostname to use
+   -sni string[]                tls sni hostname to use
    -min-version string          minimum tls version to accept (ssl30,tls10,tls11,tls12,tls13)
    -max-version string          maximum tls version to accept (ssl30,tls10,tls11,tls12,tls13)
-   -tc, -tls-chain              display tls chain in json output
+   -ac, -all-ciphers            send all ciphers as accepted inputs
+   -cert, -certificate          include certificates in json output (PEM format)
+   -tc, -tls-chain              include certificates chain in json output
    -vc, -verify-cert            enable verification of server certificate
 
 OPTIMIZATIONS:
    -c, -concurrency int  number of concurrent threads to process (default 300)
    -timeout int          tls connection timeout in seconds (default 5)
+   -retries int          number of retries to perform for failures (default 3)
 
 OUTPUT:
    -o, -output string  file to write output to
@@ -316,6 +321,22 @@ self-signed.badssl.com:443 [self-signed]
 expired.badssl.com:443 [expired]
 ```
 
+### [JARM](https://engineering.salesforce.com/easily-identify-malicious-servers-on-the-internet-with-jarm-e095edac525a/) TLS Fingerprint
+
+```console
+$ echo hackerone.com | tlsx -jarm -silent
+
+hackerone.com:443 [29d3dd00029d29d00042d43d00041d5de67cc9954cc85372523050f20b5007]
+```
+
+### [JA3](https://github.com/salesforce/ja3) TLS Fingerprint
+
+```console
+$ echo hackerone.com | tlsx -ja3 -silent
+
+hackerone.com:443 [20c9baf81bfe96ff89722899e75d0190]
+```
+
 ### JSON Output
 
 **tlsx** does support multiple probe flags to query specific data, but all the information is always available in JSON format, for automation and post processing using `-json` output is most convenient option to use.

cmd/tlsx/main.go

@@ -62,6 +62,7 @@ func readFlags() error {
 		flagSet.BoolVar(&options.Cipher, "cipher", false, "display used cipher"),
 		flagSet.StringVar(&options.Hash, "hash", "", "display certificate fingerprint hashes (md5,sha1,sha256)"),
 		flagSet.BoolVar(&options.Jarm, "jarm", false, "display jarm fingerprint hash"),
+		flagSet.BoolVar(&options.Ja3, "ja3", false, "display ja3 fingerprint hash (using ztls)"),
 		flagSet.BoolVarP(&options.ProbeStatus, "probe-status", "tps", false, "display tls probe status"),
 	)
 
@@ -79,13 +80,16 @@ func readFlags() error {
 		flagSet.StringSliceVar(&options.ServerName, "sni", nil, "tls sni hostname to use", goflags.FileCommaSeparatedStringSliceOptions),
 		flagSet.StringVar(&options.MinVersion, "min-version", "", "minimum tls version to accept (ssl30,tls10,tls11,tls12,tls13)"),
 		flagSet.StringVar(&options.MaxVersion, "max-version", "", "maximum tls version to accept (ssl30,tls10,tls11,tls12,tls13)"),
-		flagSet.BoolVarP(&options.TLSChain, "tls-chain", "tc", false, "display tls chain in json output"),
+		flagSet.BoolVarP(&options.AllCiphers, "all-ciphers", "ac", false, "send all ciphers as accepted inputs"),
+		flagSet.BoolVarP(&options.Cert, "certificate", "cert", false, "include certificates in json output (PEM format)"),
+		flagSet.BoolVarP(&options.TLSChain, "tls-chain", "tc", false, "include certificates chain in json output"),
 		flagSet.BoolVarP(&options.VerifyServerCertificate, "verify-cert", "vc", false, "enable verification of server certificate"),
 	)
 
 	flagSet.CreateGroup("optimizations", "Optimizations",
 		flagSet.IntVarP(&options.Concurrency, "concurrency", "c", 300, "number of concurrent threads to process"),
 		flagSet.IntVar(&options.Timeout, "timeout", 5, "tls connection timeout in seconds"),
+		flagSet.IntVar(&options.Retries, "retries", 3, "number of retries to perform for failures"),
 	)
 
 	flagSet.CreateGroup("output", "Output",

internal/runner/banner.go

@@ -17,12 +17,15 @@ var banner = fmt.Sprintf(`
    |_| |____|___/_/\_\	%s
 `, version)
 
-var version = "v0.0.4"
+var version = "v0.0.5"
 
 // validateOptions validates the provided options for crawler
 func (r *Runner) validateOptions() error {
 	r.hasStdin = fileutil.HasStdin()
 
+	if r.options.Retries == 0 {
+		r.options.Retries = 1
+	}
 	probeSpecified := r.options.SO || r.options.TLSVersion || r.options.Cipher || r.options.Expired || r.options.SelfSigned || r.options.Hash != "" || r.options.Jarm || r.options.MisMatched
 	if r.options.RespOnly && probeSpecified {
 		return errors.New("resp-only flag can only be used with san and cn flags")
@@ -40,7 +43,7 @@ func (r *Runner) validateOptions() error {
 	if r.options.CertsOnly && !(r.options.ScanMode == "ztls" || r.options.ScanMode == "auto") {
 		return errors.New("scan-mode must be ztls or auto with certs-only option")
 	}
-	if r.options.CertsOnly {
+	if r.options.CertsOnly || r.options.Ja3 {
 		r.options.ScanMode = "ztls" // force setting ztls when using certs-only
 	}
 	if r.options.Verbose {

pkg/output/output.go

@@ -151,7 +151,7 @@ func (w *StandardWriter) formatStandard(output *clients.Response) ([]byte, error
 		builder.WriteString(w.aurora.Blue(output.ServerName).String())
 		builder.WriteString("]")
 	}
-	if w.options.SO  && len(cert.SubjectOrg) > 0 {
+	if w.options.SO && len(cert.SubjectOrg) > 0 {
 		builder.WriteString(" [")
 		builder.WriteString(w.aurora.BrightYellow(strings.Join(cert.SubjectOrg, ",")).String())
 		builder.WriteString("]")
@@ -205,6 +205,12 @@ func (w *StandardWriter) formatStandard(output *clients.Response) ([]byte, error
 		builder.WriteString("]")
 	}
 
+	if w.options.Ja3 && output.Ja3Hash != "" {
+		builder.WriteString(" [")
+		builder.WriteString(w.aurora.Magenta(output.Ja3Hash).String())
+		builder.WriteString("]")
+	}
+
 	outputdata := builder.Bytes()
 	return outputdata, nil
 }

pkg/tlsx/clients/clients.go

@@ -6,6 +6,7 @@ import (
 	"crypto/sha1"
 	"crypto/sha256"
 	"encoding/hex"
+	"encoding/pem"
 	"math"
 	"strings"
 	"time"
@@ -39,6 +40,8 @@ type Options struct {
 	JSON bool
 	// TLSChain enables printing TLS chain information to output
 	TLSChain bool
+	// AllCiphers enables sending all ciphers as client
+	AllCiphers bool
 	// ProbeStatus enables writing of errors with json output
 	ProbeStatus bool
 	// CertsOnly enables early SSL termination using ztls flag
@@ -49,6 +52,8 @@ type Options struct {
 	Silent bool
 	// NoColor disables coloring of CLI output
 	NoColor bool
+	// Retries is the number of times to retry TLS connection
+	Retries int
 	// Timeout is the number of seconds to wait for connection
 	Timeout int
 	// Concurrency is the number of concurrent threads to process
@@ -92,6 +97,10 @@ type Options struct {
 	Hash string
 	// Jarm calculate jarm fingerprinting with multiple probes
 	Jarm bool
+	// Cert displays certificate in pem format
+	Cert bool
+	// Ja3 displays ja3 fingerprint hash
+	Ja3 bool
 
 	// Fastdialer is a fastdialer dialer instance
 	Fastdialer *fastdialer.Dialer
@@ -124,6 +133,7 @@ type Response struct {
 	// Chain is the chain of certificates
 	Chain      []*CertificateResponse `json:"chain,omitempty"`
 	JarmHash   string                 `json:"jarm_hash,omitempty"`
+	Ja3Hash    string                 `json:"ja3_hash,omitempty"`
 	ServerName string                 `json:"sni,omitempty"`
 }
 
@@ -157,6 +167,8 @@ type CertificateResponse struct {
 	Emails []string `json:"emails,omitempty"`
 	// FingerprintHash is the hashes for certificate
 	FingerprintHash CertificateResponseFingerprintHash `json:"fingerprint_hash,omitempty"`
+	// Certificate is the raw certificate in PEM format
+	Certificate string `json:"certificate,omitempty"`
 }
 
 // CertificateDistinguishedName is a distinguished certificate name
@@ -247,6 +259,15 @@ func IsMisMatchedCert(host string, names []string) bool {
 	return true
 }
 
+// PemEncode encodes a raw certificate to PEM format.
+func PemEncode(cert []byte) string {
+	var buf bytes.Buffer
+	if err := pem.Encode(&buf, &pem.Block{Type: "CERTIFICATE", Bytes: cert}); err != nil {
+		return ""
+	}
+	return buf.String()
+}
+
 type ConnectOptions struct {
 	SNI string
 }

pkg/tlsx/tls/tls.go

@@ -57,6 +57,9 @@ func New(options *clients.Options) (*Client, error) {
 		options: options,
 	}
 
+	if options.AllCiphers {
+		c.tlsConfig.CipherSuites = allCiphers
+	}
 	if len(options.Ciphers) > 0 {
 		if customCiphers, err := toTLSCiphers(options.Ciphers); err != nil {
 			return nil, errors.Wrap(err, "could not get tls ciphers")
@@ -156,18 +159,18 @@ func (c *Client) ConnectWithOptions(hostname, port string, options clients.Conne
 		Version:             tlsVersion,
 		Cipher:              tlsCipher,
 		TLSConnection:       "ctls",
-		CertificateResponse: convertCertificateToResponse(hostname, leafCertificate),
+		CertificateResponse: c.convertCertificateToResponse(hostname, leafCertificate),
 		ServerName:          config.ServerName,
 	}
 	if c.options.TLSChain {
 		for _, cert := range certificateChain {
-			response.Chain = append(response.Chain, convertCertificateToResponse(hostname, cert))
+			response.Chain = append(response.Chain, c.convertCertificateToResponse(hostname, cert))
 		}
 	}
 	return response, nil
 }
 
-func convertCertificateToResponse(hostname string, cert *x509.Certificate) *clients.CertificateResponse {
+func (c *Client) convertCertificateToResponse(hostname string, cert *x509.Certificate) *clients.CertificateResponse {
 	response := &clients.CertificateResponse{
 		SubjectAN:  cert.DNSNames,
 		Emails:     cert.EmailAddresses,
@@ -196,6 +199,9 @@ func convertCertificateToResponse(hostname string, cert *x509.Certificate) *clie
 	} else {
 		response.SubjectDN = cert.Subject.String()
 	}
+	if c.options.Cert {
+		response.Certificate = clients.PemEncode(cert.Raw)
+	}
 	return response
 }
 

pkg/tlsx/tls/utils.go

@@ -5,6 +5,14 @@ import (
 	"fmt"
 )
 
+var allCiphers []uint16
+
+func init() {
+	for _, cipher := range tlsCiphers {
+		allCiphers = append(allCiphers, cipher)
+	}
+}
+
 func toTLSCiphers(items []string) ([]uint16, error) {
 	var convertedCiphers []uint16
 	for _, item := range items {

pkg/tlsx/tlsx.go

@@ -48,14 +48,26 @@ func (s *Service) Connect(host, port string) (*clients.Response, error) {
 
 // Connect connects to the input with custom options
 func (s *Service) ConnectWithOptions(host, port string, options clients.ConnectOptions) (*clients.Response, error) {
-	resp, err := s.client.ConnectWithOptions(host, port, options)
+	var resp *clients.Response
+	var err error
+
+	for i := 0; i < s.options.Retries; i++ {
+		if resp, err = s.client.ConnectWithOptions(host, port, options); resp != nil {
+			err = nil
+			break
+		}
+	}
+	if resp == nil && err == nil {
+		return nil, errors.New("no response returned for connection")
+	}
 	if err != nil {
 		wrappedErr := errors.Wrap(err, "could not connect to host")
 		if s.options.ProbeStatus {
 			return &clients.Response{Host: host, Port: port, Error: err.Error(), ProbeStatus: false, ServerName: options.SNI}, wrappedErr
 		}
 		return nil, wrappedErr
 	}
+
 	if s.options.Jarm {
 		port, _ := strconv.Atoi(port)
 		timeout := time.Duration(s.options.Timeout) * time.Second