Skip to content

Commit

Permalink
feat(ct): introduce Certificate Transparency Log parsing
Browse files Browse the repository at this point in the history
  • Loading branch information
@stuchl4n3k
stuchl4n3k committed Nov 10, 2023
1 parent 6269834 commit 6978387
Show file tree
Hide file tree
Showing 5 changed files with 299 additions and 36 deletions.
66 changes: 35 additions & 31 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ Feature set:
- [x] Unobtrusive human-readable CLI output as well as machine readable JSON
- [x] Supports multiple domains on the input
- [x] Colorized output
- [x] Resolves domains in HTTP headers
- [x] Parses domains in HTTP headers
- [x] Parses domains in Certificate Transparency logs
- [x] Parses IPs found in SPF record
- [x] Looks up BGP AS for each discovered IP
- [x] Looks up GeoIP record for each discovered IP
Expand All @@ -49,27 +50,27 @@ for _, res := range resolutions {
```
+------------+
| |
+------+ Udig +------------+
Delegates: | | | |
| +------------+ |
|* |*
+------------------+ +------------+
| DomainResolver | | IPResolver |
+----------------------> +------------------+ +------------+
| ^ ^ ^ ^ ^
Implements: | +-----+ | | | +-------+
| | | | | |
+-------------+ +-------------+ +--------------+ +---------------+ +-------------+ +---------------+
| DNSResolver | | TLSResolver | | HTTPResolver | | WhoisResolver | | BGPResolver | | GeoipResolver |
+-------------+ +-------------+ +--------------+ +---------------+ +-------------+ +---------------+
| | | | | |
| | | | | |
Produces: | | | | | |
| | | | | |
|* |* |* |* |* |*
+-----------+ +----------------+ +------------+ +--------------+ +----------+ +-------------+
| DNSRecord | | TLSCertificate | | HTTPHeader | | WhoisContact | | ASRecord | | GeoipRecord |
+-----------+ +----------------+ +------------+ +--------------+ +----------+ +-------------+
+------+ Udig +-----------------------------------+
Delegates: | | | |
| +------------+ |
|* |*
+------------------+ +------------+
| DomainResolver | | IPResolver |
+----------------------> +------------------+ <------------------+ +------------+
| ^ ^ ^ | ^ ^
Implements: | +-----+ | | | | +-------+
| | | | | | |
+-------------+ +-------------+ +--------------+ +---------------+ +------------+ +-------------+ +---------------+
| DNSResolver | | TLSResolver | | HTTPResolver | | WhoisResolver | | CTResolver | | BGPResolver | | GeoipResolver |
+-------------+ +-------------+ +--------------+ +---------------+ +------------+ +-------------+ +---------------+
| | | | | | |
| | | | | | |
Produces: | | | | | | |
| | | | | | |
|* |* |* |* |* |* |*
+-----------+ +----------------+ +------------+ +--------------+ +-------+ +----------+ +-------------+
| DNSRecord | | TLSCertificate | | HTTPHeader | | WhoisContact | | CTLog | | ASRecord | | GeoipRecord |
+-----------+ +----------------+ +------------+ +--------------+ +-------+ +----------+ +-------------+
```

Expand All @@ -88,19 +89,22 @@ This will also download the latest GeoIP database (IPLocation-lite).
### Usage

```bash
udig [-h|--help] [-v|--version] [-V|--verbose] [-s|--strict] [--json]
[-d|--domain "<value>" [-d|--domain "<value>" ...]]
udig [-h|--help] [-v|--version] [-V|--verbose] [-s|--strict]
[-d|--domain "<value>"] [--ct:expired] [--ct:from "<value>"]
[--json]

ÜberDig - dig on steroids v1.4 by stuchl4n3k
ÜberDig - dig on steroids v1.5 by stuchl4n3k

Arguments:

-h --help Print help information
-v --version Print version and exit
-V --verbose Be more verbose
-s --strict Strict domain relation (TLD match)
--json Output payloads as JSON objects
-d --domain Domain to resolve
-h --help Print help information
-v --version Print version and exit
-V --verbose Be more verbose
-s --strict Strict domain relation (TLD match)
-d --domain Domain to resolve
--ct:expired Collect expired CT logs
--ct:from Date to collect logs from. Default: 1 year ago (2022-11-10)
--json Output payloads as JSON objects
```

### Demo
Expand Down
47 changes: 44 additions & 3 deletions api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,11 @@ package udig

import (
"crypto/x509"
"github.com/domainr/whois"
"github.com/miekg/dns"
"net/http"
"time"

"github.com/domainr/whois"
"github.com/miekg/dns"
)

/////////////////////////////////////////
Expand Down Expand Up @@ -33,6 +34,9 @@ const (
// TypeHTTP is a type of all HTTP resolutions.
TypeHTTP ResolutionType = "HTTP"

// TypeCT is a type of all CT resolutions.
TypeCT ResolutionType = "CT"

// TypeBGP is a type of all BGP resolutions.
TypeBGP ResolutionType = "BGP"

Expand All @@ -41,7 +45,7 @@ const (
)

// Udig is a high-level facade for domain resolution which:
// 1. delegates work to specific resolvers
// 1. delegates work to specific resolvers
// 2. deals with domain crawling
// 3. caches intermediate results and summarizes the outputs
type Udig interface {
Expand Down Expand Up @@ -202,6 +206,43 @@ type HTTPHeader struct {
Value []string
}

/////////////////////////////////////////
// CT
/////////////////////////////////////////

// CTResolver is a Resolver responsible for resolution of a given domain
// to a list of CT logs.
type CTResolver struct {
DomainResolver
Client *http.Client
cachedResults map[string]*CTResolution
}

// CTResolution is a certificate transparency project resolution, which yields a CT log.
type CTResolution struct {
*ResolutionBase
Logs []CTAggregatedLog
}

// CTAggregatedLog is a wrapper of a CT log that is aggregated over all logs
// with the same CN in time.
type CTAggregatedLog struct {
CTLog
FirstSeen string
LastSeen string
}

// CTLog is a wrapper for attributes of interest that appear in the CT log.
// The json mapping comes from crt.sh API schema.
type CTLog struct {
Id int64 `json:"id"`
IssuerName string `json:"issuer_name"`
NameValue string `json:"name_value"`
LoggedAt string `json:"entry_timestamp"`
NotBefore string `json:"not_before"`
NotAfter string `json:"not_after"`
}

/////////////////////////////////////////
// BGP
/////////////////////////////////////////
Expand Down
29 changes: 27 additions & 2 deletions cmd/udig/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"fmt"
"net/url"
"os"
"time"

"github.com/akamensky/argparse"
"github.com/miekg/dns"
Expand All @@ -13,7 +14,7 @@ import (

const (
prog = "udig"
version = "1.4"
version = "1.5"
author = "stuchl4n3k"
description = "ÜberDig - dig on steroids v" + version + " by " + author
)
Expand Down Expand Up @@ -65,6 +66,12 @@ func resolve(domain string) {
}
break

case udig.TypeCT:
for _, ctLog := range (res).(*udig.CTResolution).Logs {
udig.LogInfo("%s: %s -> %s", res.Type(), res.Query(), formatPayload(&ctLog))
}
break

case udig.TypeBGP:
for _, as := range (res).(*udig.BGPResolution).Records {
udig.LogInfo("%s: %s -> %s", res.Type(), res.Query(), formatPayload(&as))
Expand Down Expand Up @@ -105,8 +112,18 @@ func main() {
printVersion := parser.Flag("v", "version", &argparse.Options{Required: false, Help: "Print version and exit"})
beVerbose := parser.Flag("V", "verbose", &argparse.Options{Required: false, Help: "Be more verbose"})
beStrict := parser.Flag("s", "strict", &argparse.Options{Required: false, Help: "Strict domain relation (TLD match)"})
jsonOutput := parser.Flag("", "json", &argparse.Options{Required: false, Help: "Output payloads as JSON objects"})
domain := parser.String("d", "domain", &argparse.Options{Required: false, Help: "Domain to resolve"})
ctExpired := parser.Flag("", "ct:expired", &argparse.Options{Required: false, Help: "Collect expired CT logs"})
ctFrom := parser.String("", "ct:from", &argparse.Options{
Required: false,
Help: "Date to collect logs from",
Default: fmt.Sprintf("1 year ago (%s)", udig.CTLogFrom),
Validate: func(args []string) error {
_, err := time.Parse("2006-01-02", args[0])
return err
},
})
jsonOutput := parser.Flag("", "json", &argparse.Options{Required: false, Help: "Output payloads as JSON objects"})

err := parser.Parse(os.Args)
if err != nil {
Expand All @@ -132,6 +149,14 @@ func main() {
udig.IsDomainRelated = udig.StrictDomainRelation
}

if *ctExpired {
udig.CTExclude = ""
}

if *ctFrom != "" {
udig.CTLogFrom = *ctFrom
}

outputJson = *jsonOutput

fmt.Println(banner)
Expand Down
Loading

0 comments on commit 6978387

Please sign in to comment.