go: save all breaches as cache, search by name

* also switch addedDate column to string temporarily, until saving
  yy-mm-dd as time is solved...

ent/hibp.go

@@ -27,7 +27,7 @@ type HIBP struct {
 	// Domain holds the value of the "domain" field.
 	Domain string `json:"domain,omitempty"`
 	// BreachDate holds the value of the "breach_date" field.
-	BreachDate time.Time `json:"breach_date,omitempty"`
+	BreachDate string `json:"breach_date,omitempty"`
 	// AddedDate holds the value of the "added_date" field.
 	AddedDate time.Time `json:"added_date,omitempty"`
 	// ModifiedDate holds the value of the "modified_date" field.
@@ -92,9 +92,9 @@ func (*HIBP) scanValues(columns []string) ([]any, error) {
 			values[i] = new(sql.NullBool)
 		case hibp.FieldPwnCount:
 			values[i] = new(sql.NullInt64)
-		case hibp.FieldName, hibp.FieldTitle, hibp.FieldDomain, hibp.FieldDescription, hibp.FieldLogoPath:
+		case hibp.FieldName, hibp.FieldTitle, hibp.FieldDomain, hibp.FieldBreachDate, hibp.FieldDescription, hibp.FieldLogoPath:
 			values[i] = new(sql.NullString)
-		case hibp.FieldBreachDate, hibp.FieldAddedDate, hibp.FieldModifiedDate:
+		case hibp.FieldAddedDate, hibp.FieldModifiedDate:
 			values[i] = new(sql.NullTime)
 		case hibp.FieldID:
 			values[i] = new(uuid.UUID)
@@ -140,10 +140,10 @@ func (h *HIBP) assignValues(columns []string, values []any) error {
 				h.Domain = value.String
 			}
 		case hibp.FieldBreachDate:
-			if value, ok := values[i].(*sql.NullTime); !ok {
+			if value, ok := values[i].(*sql.NullString); !ok {
 				return fmt.Errorf("unexpected type %T for field breach_date", values[i])
 			} else if value.Valid {
-				h.BreachDate = value.Time
+				h.BreachDate = value.String
 			}
 		case hibp.FieldAddedDate:
 			if value, ok := values[i].(*sql.NullTime); !ok {
@@ -279,7 +279,7 @@ func (h *HIBP) String() string {
 	builder.WriteString(h.Domain)
 	builder.WriteString(", ")
 	builder.WriteString("breach_date=")
-	builder.WriteString(h.BreachDate.Format(time.ANSIC))
+	builder.WriteString(h.BreachDate)
 	builder.WriteString(", ")
 	builder.WriteString("added_date=")
 	builder.WriteString(h.AddedDate.Format(time.ANSIC))

ent/hibp/where.go

@@ -72,7 +72,7 @@ func Domain(v string) predicate.HIBP {
 }
 
 // BreachDate applies equality check predicate on the "breach_date" field. It's identical to BreachDateEQ.
-func BreachDate(v time.Time) predicate.HIBP {
+func BreachDate(v string) predicate.HIBP {
 	return predicate.HIBP(sql.FieldEQ(FieldBreachDate, v))
 }
 
@@ -327,45 +327,70 @@ func DomainContainsFold(v string) predicate.HIBP {
 }
 
 // BreachDateEQ applies the EQ predicate on the "breach_date" field.
-func BreachDateEQ(v time.Time) predicate.HIBP {
+func BreachDateEQ(v string) predicate.HIBP {
 	return predicate.HIBP(sql.FieldEQ(FieldBreachDate, v))
 }
 
 // BreachDateNEQ applies the NEQ predicate on the "breach_date" field.
-func BreachDateNEQ(v time.Time) predicate.HIBP {
+func BreachDateNEQ(v string) predicate.HIBP {
 	return predicate.HIBP(sql.FieldNEQ(FieldBreachDate, v))
 }
 
 // BreachDateIn applies the In predicate on the "breach_date" field.
-func BreachDateIn(vs ...time.Time) predicate.HIBP {
+func BreachDateIn(vs ...string) predicate.HIBP {
 	return predicate.HIBP(sql.FieldIn(FieldBreachDate, vs...))
 }
 
 // BreachDateNotIn applies the NotIn predicate on the "breach_date" field.
-func BreachDateNotIn(vs ...time.Time) predicate.HIBP {
+func BreachDateNotIn(vs ...string) predicate.HIBP {
 	return predicate.HIBP(sql.FieldNotIn(FieldBreachDate, vs...))
 }
 
 // BreachDateGT applies the GT predicate on the "breach_date" field.
-func BreachDateGT(v time.Time) predicate.HIBP {
+func BreachDateGT(v string) predicate.HIBP {
 	return predicate.HIBP(sql.FieldGT(FieldBreachDate, v))
 }
 
 // BreachDateGTE applies the GTE predicate on the "breach_date" field.
-func BreachDateGTE(v time.Time) predicate.HIBP {
+func BreachDateGTE(v string) predicate.HIBP {
 	return predicate.HIBP(sql.FieldGTE(FieldBreachDate, v))
 }
 
 // BreachDateLT applies the LT predicate on the "breach_date" field.
-func BreachDateLT(v time.Time) predicate.HIBP {
+func BreachDateLT(v string) predicate.HIBP {
 	return predicate.HIBP(sql.FieldLT(FieldBreachDate, v))
 }
 
 // BreachDateLTE applies the LTE predicate on the "breach_date" field.
-func BreachDateLTE(v time.Time) predicate.HIBP {
+func BreachDateLTE(v string) predicate.HIBP {
 	return predicate.HIBP(sql.FieldLTE(FieldBreachDate, v))
 }
 
+// BreachDateContains applies the Contains predicate on the "breach_date" field.
+func BreachDateContains(v string) predicate.HIBP {
+	return predicate.HIBP(sql.FieldContains(FieldBreachDate, v))
+}
+
+// BreachDateHasPrefix applies the HasPrefix predicate on the "breach_date" field.
+func BreachDateHasPrefix(v string) predicate.HIBP {
+	return predicate.HIBP(sql.FieldHasPrefix(FieldBreachDate, v))
+}
+
+// BreachDateHasSuffix applies the HasSuffix predicate on the "breach_date" field.
+func BreachDateHasSuffix(v string) predicate.HIBP {
+	return predicate.HIBP(sql.FieldHasSuffix(FieldBreachDate, v))
+}
+
+// BreachDateEqualFold applies the EqualFold predicate on the "breach_date" field.
+func BreachDateEqualFold(v string) predicate.HIBP {
+	return predicate.HIBP(sql.FieldEqualFold(FieldBreachDate, v))
+}
+
+// BreachDateContainsFold applies the ContainsFold predicate on the "breach_date" field.
+func BreachDateContainsFold(v string) predicate.HIBP {
+	return predicate.HIBP(sql.FieldContainsFold(FieldBreachDate, v))
+}
+
 // AddedDateEQ applies the EQ predicate on the "added_date" field.
 func AddedDateEQ(v time.Time) predicate.HIBP {
 	return predicate.HIBP(sql.FieldEQ(FieldAddedDate, v))

ent/hibp_create.go

@@ -41,8 +41,8 @@ func (hc *HIBPCreate) SetDomain(s string) *HIBPCreate {
 }
 
 // SetBreachDate sets the "breach_date" field.
-func (hc *HIBPCreate) SetBreachDate(t time.Time) *HIBPCreate {
-	hc.mutation.SetBreachDate(t)
+func (hc *HIBPCreate) SetBreachDate(s string) *HIBPCreate {
+	hc.mutation.SetBreachDate(s)
 	return hc
 }
 
@@ -365,7 +365,7 @@ func (hc *HIBPCreate) createSpec() (*HIBP, *sqlgraph.CreateSpec) {
 		_node.Domain = value
 	}
 	if value, ok := hc.mutation.BreachDate(); ok {
-		_spec.SetField(hibp.FieldBreachDate, field.TypeTime, value)
+		_spec.SetField(hibp.FieldBreachDate, field.TypeString, value)
 		_node.BreachDate = value
 	}
 	if value, ok := hc.mutation.AddedDate(); ok {

ent/hibp_update.go

@@ -38,8 +38,8 @@ func (hu *HIBPUpdate) SetTitle(s string) *HIBPUpdate {
 }
 
 // SetBreachDate sets the "breach_date" field.
-func (hu *HIBPUpdate) SetBreachDate(t time.Time) *HIBPUpdate {
-	hu.mutation.SetBreachDate(t)
+func (hu *HIBPUpdate) SetBreachDate(s string) *HIBPUpdate {
+	hu.mutation.SetBreachDate(s)
 	return hu
 }
 
@@ -274,7 +274,7 @@ func (hu *HIBPUpdate) sqlSave(ctx context.Context) (n int, err error) {
 		_spec.SetField(hibp.FieldTitle, field.TypeString, value)
 	}
 	if value, ok := hu.mutation.BreachDate(); ok {
-		_spec.SetField(hibp.FieldBreachDate, field.TypeTime, value)
+		_spec.SetField(hibp.FieldBreachDate, field.TypeString, value)
 	}
 	if value, ok := hu.mutation.AddedDate(); ok {
 		_spec.SetField(hibp.FieldAddedDate, field.TypeTime, value)
@@ -382,8 +382,8 @@ func (huo *HIBPUpdateOne) SetTitle(s string) *HIBPUpdateOne {
 }
 
 // SetBreachDate sets the "breach_date" field.
-func (huo *HIBPUpdateOne) SetBreachDate(t time.Time) *HIBPUpdateOne {
-	huo.mutation.SetBreachDate(t)
+func (huo *HIBPUpdateOne) SetBreachDate(s string) *HIBPUpdateOne {
+	huo.mutation.SetBreachDate(s)
 	return huo
 }
 
@@ -648,7 +648,7 @@ func (huo *HIBPUpdateOne) sqlSave(ctx context.Context) (_node *HIBP, err error)
 		_spec.SetField(hibp.FieldTitle, field.TypeString, value)
 	}
 	if value, ok := huo.mutation.BreachDate(); ok {
-		_spec.SetField(hibp.FieldBreachDate, field.TypeTime, value)
+		_spec.SetField(hibp.FieldBreachDate, field.TypeString, value)
 	}
 	if value, ok := huo.mutation.AddedDate(); ok {
 		_spec.SetField(hibp.FieldAddedDate, field.TypeTime, value)

ent/migrate/schema.go

@@ -36,7 +36,7 @@ var (
 		{Name: "name", Type: field.TypeString, Unique: true},
 		{Name: "title", Type: field.TypeString, Unique: true},
 		{Name: "domain", Type: field.TypeString},
-		{Name: "breach_date", Type: field.TypeTime},
+		{Name: "breach_date", Type: field.TypeString},
 		{Name: "added_date", Type: field.TypeTime},
 		{Name: "modified_date", Type: field.TypeTime},
 		{Name: "pwn_count", Type: field.TypeInt},

ent/mutation.go

@@ -612,7 +612,7 @@ type HIBPMutation struct {
 	name                    *string
 	title                   *string
 	domain                  *string
-	breach_date             *time.Time
+	breach_date             *string
 	added_date              *time.Time
 	modified_date           *time.Time
 	pwn_count               *int
@@ -848,12 +848,12 @@ func (m *HIBPMutation) ResetDomain() {
 }
 
 // SetBreachDate sets the "breach_date" field.
-func (m *HIBPMutation) SetBreachDate(t time.Time) {
-	m.breach_date = &t
+func (m *HIBPMutation) SetBreachDate(s string) {
+	m.breach_date = &s
 }
 
 // BreachDate returns the value of the "breach_date" field in the mutation.
-func (m *HIBPMutation) BreachDate() (r time.Time, exists bool) {
+func (m *HIBPMutation) BreachDate() (r string, exists bool) {
 	v := m.breach_date
 	if v == nil {
 		return
@@ -864,7 +864,7 @@ func (m *HIBPMutation) BreachDate() (r time.Time, exists bool) {
 // OldBreachDate returns the old "breach_date" field's value of the HIBP entity.
 // If the HIBP object wasn't provided to the builder, the object is fetched from the database.
 // An error is returned if the mutation operation is not UpdateOne, or the database query fails.
-func (m *HIBPMutation) OldBreachDate(ctx context.Context) (v time.Time, err error) {
+func (m *HIBPMutation) OldBreachDate(ctx context.Context) (v string, err error) {
 	if !m.op.Is(OpUpdateOne) {
 		return v, errors.New("OldBreachDate is only allowed on UpdateOne operations")
 	}
@@ -1610,7 +1610,7 @@ func (m *HIBPMutation) SetField(name string, value ent.Value) error {
 		m.SetDomain(v)
 		return nil
 	case hibp.FieldBreachDate:
-		v, ok := value.(time.Time)
+		v, ok := value.(string)
 		if !ok {
 			return fmt.Errorf("unexpected type %T for field %s", value, name)
 		}

ent/schema/hibp.go

@@ -62,16 +62,17 @@ func (HIBP) Fields() []ent.Field {
 		field.UUID("id", uuid.UUID{}).
 			Unique().
 			Immutable(),
-		// Unique but may change.
+		// Unique and permanent.
 		field.String("name").
 			NotEmpty().
 			Unique().
 			Immutable(),
+		// Unique but may change.
 		field.String("title").
 			Unique(),
 		field.String("domain").
 			Immutable(),
-		field.Time("breach_date"),
+		field.String("breach_date"),
 		// precision to the minute.
 		field.Time("added_date"),
 		field.Time("modified_date"),

modules/hibp/context.go

@@ -0,0 +1,7 @@
+// Copyright 2023 wanderer <a_mirre at utb dot cz>
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package hibp
+
+// CtxKey serves as a key to context values for this package.
+type CtxKey struct{}

modules/hibp/error.go

@@ -0,0 +1,15 @@
+// Copyright 2023 wanderer <a_mirre at utb dot cz>
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package hibp
+
+import "errors"
+
+var (
+	ErrAuthKeyCheckValue     = errors.New(authKeyCheckValue)
+	ErrRateLimited           = errors.New("We have been rate limited")
+	ErrBreachNotFound        = errors.New("Breach not found")
+	ErrBreachNotSingular     = errors.New("Multiple breaches for one name")
+	ErrFailedToQueryBreaches = errors.New("Failed to query breaches")
+	ErrNoBreachesToSave      = errors.New("No breaches to save")
+)

modules/hibp/hibp.go

@@ -4,15 +4,18 @@
 package hibp
 
 import (
+	"context"
 	"encoding/json"
-	"errors"
 	"io"
 	"log"
 	"net/http"
 	"os"
 	"time"
 
+	"git.dotya.ml/mirre-mt/pcmt/ent"
+	"git.dotya.ml/mirre-mt/pcmt/ent/hibp"
 	"git.dotya.ml/mirre-mt/pcmt/ent/schema"
+	"git.dotya.ml/mirre-mt/pcmt/slogging"
 	"golang.org/x/exp/slog"
 )
 
@@ -54,9 +57,6 @@ const (
 var (
 	apiKey = os.Getenv("PCMT_HIBP_API_KEY")
 	client = &http.Client{Timeout: reqTmOut}
-
-	ErrAuthKeyCheckValue = errors.New(authKeyCheckValue)
-	ErrRateLimited       = errors.New("We have been rate limited")
 )
 
 // SubscriptionStatus models https://haveibeenpwned.com/API/v3#SubscriptionStatus.
@@ -184,6 +184,89 @@ func GetAllBreachesForAccount(account string) ([]BreachName, error) {
 	return bn, nil
 }
 
+// GetBreachesForBreachNames retrieves HIBP breaches from the database for a
+// list of names.
+func GetBreachesForBreachNames(ctx context.Context, client *ent.Client, names []string) ([]*ent.HIBP, error) {
+	slogger := ctx.Value(CtxKey{}).(*slogging.Slogger)
+	log := *slogger
+
+	log.Logger = log.Logger.With(
+		slog.Group("pcmt extra", slog.String("module", "modules/hibp")),
+	)
+
+	hs := make([]*ent.HIBP, 0)
+
+	for _, name := range names {
+		b, err := client.HIBP.
+			Query().
+			Where(hibp.NameEQ(name)).
+			Only(ctx)
+		if err != nil {
+			switch {
+			case ent.IsNotFound(err):
+				log.Warnf("breach not found by name %q: %q", name, err)
+				return nil, ErrBreachNotFound
+
+			case ent.IsNotSingular(err):
+				log.Warnf("multiple breaches returned for name %q: %q", name, err)
+				return nil, ErrBreachNotSingular
+
+			case err != nil:
+				log.Warn("failed to query breach by name", "error", err, "name requested", name)
+				return nil, ErrFailedToQueryBreaches
+
+			default:
+				return nil, err
+			}
+		}
+
+		hs = append(hs, b)
+	}
+
+	return hs, nil
+}
+
+// SaveAllBreaches saves all breaches to DB as a cache.
+func SaveAllBreaches(ctx context.Context, client *ent.Client, breaches *[]schema.HIBPSchema) error {
+	slogger := ctx.Value(CtxKey{}).(*slogging.Slogger)
+	log := *slogger
+
+	log.Logger = log.Logger.With(
+		slog.Group("pcmt extra", slog.String("module", "modules/hibp")),
+	)
+
+	if breaches == nil {
+		return ErrNoBreachesToSave
+	}
+
+	for _, b := range *breaches {
+		_, err := client.HIBP.
+			Create().
+			SetName(b.Name).
+			SetTitle(b.Title).
+			SetDomain(b.Domain).
+			SetBreachDate(b.BreachDate).
+			SetAddedDate(b.AddedDate).
+			SetModifiedDate(b.ModifiedDate).
+			SetPwnCount(b.PwnCount).
+			SetDescription(b.Description).
+			SetDataclasses(b.DataClasses).
+			SetIsVerified(b.IsVerified).
+			SetIsFabricated(b.IsFabricated).
+			SetIsSensitive(b.IsSensitive).
+			SetIsRetired(b.IsRetired).
+			SetIsSpamList(b.IsSpamList).
+			SetIsMalware(b.IsMalware).
+			SetLogoPath(b.LogoPath).
+			Save(ctx)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func setUA(r *http.Request) {
 	r.Header.Set(headerUA, appID)
 }