Open Source Release
123 lines
4.1 KiB
Go
123 lines
4.1 KiB
Go
// Package scraper fetches component listings from supplier sites (Regard
|
|
// first; pg.pro/technosuccess.ru/DNS-shop are authenticated or anti-bot
|
|
// protected and come later) and caches them in the external_components
|
|
// table (migrations/053_external_components.sql). internal/pcbuilder reads
|
|
// that cache to offer just-in-time-sourced parts alongside the shop's own
|
|
// parts stock — this package never touches parts or talks to pcbuilder
|
|
// directly, it only knows how to fill external_components.
|
|
package scraper
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
|
|
"production/internal/pcbuilder"
|
|
)
|
|
|
|
// Item is one listing entry from a supplier site, already resolved to a
|
|
// single component type. Spec is best-effort — sites don't expose
|
|
// structured specs the way our own parts.pc_spec does, so fields are
|
|
// regex-extracted from free-text listing copy and left empty when
|
|
// unrecognized. pcbuilder.Check already treats empty spec fields as
|
|
// "nothing to compare," so a sparse Spec just means fewer compatibility
|
|
// checks apply to that item, not an error.
|
|
type Item struct {
|
|
ExternalID string
|
|
Name string
|
|
Price float64
|
|
ProductURL string
|
|
InStock bool
|
|
Spec pcbuilder.Spec
|
|
}
|
|
|
|
// Source is one supplier site's scraper. FetchCategory returns every
|
|
// listing it can find for componentType — implementations decide how to
|
|
// map that to the site's own category structure.
|
|
type Source interface {
|
|
Name() string
|
|
FetchCategory(ctx context.Context, componentType string) ([]Item, error)
|
|
}
|
|
|
|
// RefreshAll runs every source against every pcbuilder.AllTypes category and
|
|
// upserts the results into external_components, then drops rows for that
|
|
// source+type that weren't seen this run (delisted/sold-out-and-removed
|
|
// items shouldn't linger). Errors from one source/category are logged by
|
|
// the caller via the returned map rather than aborting the whole refresh —
|
|
// a broken selector on one site/category shouldn't block the others.
|
|
func RefreshAll(ctx context.Context, db *pgxpool.Pool, sources []Source) map[string]error {
|
|
errs := map[string]error{}
|
|
for _, src := range sources {
|
|
for _, componentType := range pcbuilder.AllTypes {
|
|
items, err := src.FetchCategory(ctx, componentType)
|
|
key := src.Name() + ":" + componentType
|
|
if err != nil {
|
|
errs[key] = err
|
|
continue
|
|
}
|
|
if err := upsertCategory(ctx, db, src.Name(), componentType, items); err != nil {
|
|
errs[key] = err
|
|
}
|
|
}
|
|
}
|
|
return errs
|
|
}
|
|
|
|
func upsertCategory(ctx context.Context, db *pgxpool.Pool, source, componentType string, items []Item) error {
|
|
tx, err := db.Begin(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
|
|
seen := make([]string, 0, len(items))
|
|
for _, it := range items {
|
|
specJSON, err := specToJSON(it.Spec)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
_, err = tx.Exec(ctx, `
|
|
INSERT INTO external_components (source, external_id, pc_component_type, name, price, product_url, in_stock, spec, scraped_at)
|
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, now())
|
|
ON CONFLICT (source, external_id) DO UPDATE SET
|
|
pc_component_type = EXCLUDED.pc_component_type,
|
|
name = EXCLUDED.name,
|
|
price = EXCLUDED.price,
|
|
product_url = EXCLUDED.product_url,
|
|
in_stock = EXCLUDED.in_stock,
|
|
spec = EXCLUDED.spec,
|
|
scraped_at = now()`,
|
|
source, it.ExternalID, componentType, it.Name, it.Price, it.ProductURL, it.InStock, specJSON,
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
seen = append(seen, it.ExternalID)
|
|
}
|
|
|
|
_, err = tx.Exec(ctx, `
|
|
DELETE FROM external_components
|
|
WHERE source = $1 AND pc_component_type = $2 AND NOT (external_id = ANY($3))`,
|
|
source, componentType, seen,
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return tx.Commit(ctx)
|
|
}
|
|
|
|
// StartPeriodic mirrors internal/scheduler's one-goroutine ticker shape —
|
|
// a background refresh so the cache doesn't only update when someone
|
|
// happens to hit the manual refresh endpoint.
|
|
func StartPeriodic(db *pgxpool.Pool, sources []Source, interval time.Duration) {
|
|
go func() {
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
for range ticker.C {
|
|
RefreshAll(context.Background(), db, sources)
|
|
}
|
|
}()
|
|
}
|