// Copyright 2023 wanderer // SPDX-License-Identifier: AGPL-3.0-only package hibp import ( "context" "encoding/json" "io" "net/http" "os" "time" "git.dotya.ml/mirre-mt/pcmt/ent" "git.dotya.ml/mirre-mt/pcmt/ent/hibp" "git.dotya.ml/mirre-mt/pcmt/ent/schema" "git.dotya.ml/mirre-mt/pcmt/slogging" "golang.org/x/exp/slog" ) // Subscription models the HIBP subscription struct. type Subscription struct { // The name representing the subscription being either "Pwned 1", "Pwned 2", "Pwned 3" or "Pwned 4". SubscriptionName string // A human readable sentence explaining the scope of the subscription. Description string // The date and time the current subscription ends in ISO 8601 format. SubscribedUntil time.Time // The rate limit in requests per minute. This applies to the rate the breach search by email address API can be requested. Rpm int // The size of the largest domain the subscription can search. This is expressed in the total number of breached accounts on the domain, excluding those that appear solely in spam list. DomainSearchMaxBreachedAccounts int } // BreachName is used to represent a HIBP breach name object. type BreachName struct { // Name holds the actual breach name, which in HIBP is permanently unique. Name string `json:"Name" validate:"required,Name"` } // BreachNames is a slice of BreachName objects. type BreachNames []BreachName const ( api = "https://haveibeenpwned.com/api/v3" appID = "pcmt (https://git.dotya.ml/mirre-mt/pcmt)" // set default request timeout so as not to hang forever. reqTmOut = 5 * time.Second headerUA = "user-agent" headerHIBP = "hibp-api-key" authKeyCheckValue = `Your request to the API couldn't be authorised. Check you have the right value in the "hibp-api-key" header, refer to the documentation for more: https://haveibeenpwned.com/API/v3#Authorisation` ) var ( apiKey = os.Getenv("PCMT_HIBP_API_KEY") client = &http.Client{Timeout: reqTmOut} log = slog.With( slog.Group("pcmt extra", slog.String("module", "modules/hibp")), ) ) // SubscriptionStatus models https://haveibeenpwned.com/API/v3#SubscriptionStatus. func SubscriptionStatus() (*Subscription, error) { u := api + "/subscription" req, err := http.NewRequest("GET", u, nil) if err != nil { log.Error("Could not create a new HTTP request", "error", err) } setUA(req) setAuthHeader(req) resp, err := client.Do(req) if err != nil { return nil, err } defer resp.Body.Close() body, _ := io.ReadAll(resp.Body) // bodyString := string(body) // fmt.Println("API Response as a string:\n" + bodyString) var s Subscription if err := json.Unmarshal(body, &s); err != nil { return nil, err } // fmt.Printf("Subscription struct %+v\n", s) return &Subscription{}, nil } // GetAllBreaches retrieves all breaches available in HIBP, as per // https://haveibeenpwned.com/API/v3#AllBreaches. This should be run at // start-up to populate the cache. func GetAllBreaches() (*[]schema.HIBPSchema, error) { u := api + "/breaches" req, err := http.NewRequest("GET", u, nil) if err != nil { log.Error("Could not create a new HTTP request", "error", err) return nil, err } respCh, errCh := rChans() setUA(req) slog.Info("scheduling all breaches") scheduleReq(req, &respCh, &errCh) slog.Info("scheduled all breaches") resp := <-respCh err = <-errCh defer resp.Body.Close() if err != nil { return nil, err } body, _ := io.ReadAll(resp.Body) // bodyString := string(body) // fmt.Println("API Response as a string:\n" + bodyString) ab := make([]schema.HIBPSchema, 0) if err = json.Unmarshal(body, &ab); err != nil { return nil, err } return &ab, nil } // GetAllBreachesForAccount retrieves a list of breach names for a given // account. func GetAllBreachesForAccount(account string) ([]BreachName, error) { u := api + "/breachedaccount/" + account req, err := http.NewRequest("GET", u, nil) if err != nil { log.Error("Could not create a new HTTP request", "error", err) return nil, err } respCh, errCh := rChans() setUA(req) setAuthHeader(req) scheduleReq(req, &respCh, &errCh) resp := <-respCh err = <-errCh defer resp.Body.Close() if err != nil { return nil, err } body, _ := io.ReadAll(resp.Body) if sc := resp.StatusCode; sc != 200 { // this is brittle... if string(body) == authKeyCheckValue { return nil, ErrAuthKeyCheckValue } if sc == 429 { return nil, ErrRateLimited } } bn := make([]BreachName, 0) if len(body) == 0 { return nil, nil } if err = json.Unmarshal(body, &bn); err != nil { return nil, err } return bn, nil } // GetBreachesForBreachNames retrieves HIBP breaches from the database for a // list of names. func GetBreachesForBreachNames(ctx context.Context, client *ent.Client, names []string) ([]*ent.HIBP, error) { hs := make([]*ent.HIBP, 0) for _, name := range names { b, err := client.HIBP. Query(). Where(hibp.NameEQ(name)). Only(ctx) if err != nil { switch { case ent.IsNotFound(err): log.Warn("Breach not found by name", "name", name, "error", err) return nil, ErrBreachNotFound case ent.IsNotSingular(err): log.Warn("Multiple breaches returned for name", "name", name, "error", err) return nil, ErrBreachNotSingular case err != nil: log.Warn("failed to query breach by name", "error", err, "name requested", name) return nil, ErrFailedToQueryBreaches default: return nil, err } } hs = append(hs, b) } return hs, nil } // BreachForBreachName retrieves a single HIBP breach from the database for a // given name. func BreachForBreachName(ctx context.Context, client *ent.Client, name string) (*ent.HIBP, error) { log := slog.With( slog.Group("pcmt extra", slog.String("module", "modules/hibp")), ) b, err := client.HIBP. Query(). Where(hibp.NameEQ(name)). Only(ctx) if err != nil { switch { case ent.IsNotFound(err): log.Error("Breach not found by name", "name", name, "error", err) return nil, ErrBreachNotFound case ent.IsNotSingular(err): log.Error("Multiple breaches returned for breach name", "name", name, "error", err) return nil, ErrBreachNotSingular case err != nil: log.Error("Failed to query breach by name", "error", err, "name requested", name) return nil, ErrFailedToQueryBreaches default: return nil, err } } return b, nil } // SaveAllBreaches saves all breaches to DB as a cache. func SaveAllBreaches(ctx context.Context, client *ent.Client, breaches *[]schema.HIBPSchema) error { slogger := ctx.Value(CtxKey{}).(*slogging.Slogger) log := *slogger log.Logger = log.Logger.With( slog.Group("pcmt extra", slog.String("module", "modules/hibp")), ) if breaches == nil || len(*breaches) == 0 { log.Error("Received 0 HIBP breaches / nil breaches object") return ErrNoBreachesToSave } log.Infof("HIBP API returned %d breaches, saving...", len(*breaches)) for _, b := range *breaches { _, err := client.HIBP. Create(). SetName(b.Name). SetTitle(b.Title). SetDomain(b.Domain). SetBreachDate(b.BreachDate). SetAddedDate(b.AddedDate). SetModifiedDate(b.ModifiedDate). SetPwnCount(b.PwnCount). SetDescription(b.Description). SetDataclasses(b.DataClasses). SetIsVerified(b.IsVerified). SetIsFabricated(b.IsFabricated). SetIsSensitive(b.IsSensitive). SetIsRetired(b.IsRetired). SetIsSpamList(b.IsSpamList). SetIsMalware(b.IsMalware). SetLogoPath(b.LogoPath). Save(ctx) if err != nil { log.Errorf("Could not save HIBP breaches to DB, err: %q", err) return err } } return nil } // CheckSaveAllBreaches checks if there are any in the DB and if not then // queries the API and saves what it gets. TODO: have this function consolidate // existing vs. new breaches. func CheckSaveAllBreaches(ctx context.Context, client *ent.Client) error { slogger := ctx.Value(CtxKey{}).(*slogging.Slogger) log := *slogger log.Logger = log.Logger.With( slog.Group("pcmt extra", slog.String("module", "modules/hibp")), ) log.Info("Checking if we have any HIBP breaches saved") alreadySaved, err := client.HIBP.Query().Count(ctx) switch { case err != nil: return err case alreadySaved > 0: log.Infof("There are %d HIBP breaches already, not attempting to save new breaches into DB", alreadySaved) return nil } log.Info("No HIBP breaches found in the DB, refreshing from API...") breaches, err := GetAllBreaches() if err != nil { log.Error("Could not save HIBP breaches") return err } return SaveAllBreaches(ctx, client, breaches) } func setUA(r *http.Request) { r.Header.Set(headerUA, appID) } func setAuthHeader(r *http.Request) { r.Header.Set(headerHIBP, apiKey) } func rChans() (chan *http.Response, chan error) { return make(chan *http.Response), make(chan error) }