Improve: use fetcher for core

metatube-community · Feb 14, 2024 · 700cd28 · 700cd28
1 parent 3345373
commit 700cd28
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 3 deletions.
diff --git a/common/fetch/fetch.go b/common/fetch/fetch.go
@@ -31,6 +31,9 @@ type Config struct {
  // Return error when status is not OK.
  RaiseForStatus bool
 
+ // HTTP Request timeout.
+ Timeout time.Duration
+
  // Custom HTTP Transport.
  Transport http.RoundTripper
 }
@@ -73,6 +76,9 @@ func Default(cfg *Config) *Fetcher {
  CheckRetry: retryablehttp.DefaultRetryPolicy,
  Backoff: retryablehttp.DefaultBackoff,
  }
+ if cfg.Timeout > time.Second {
+ c.HTTPClient.Timeout = cfg.Timeout
+ }
  if cfg.Transport != nil {
  c.HTTPClient.Transport = cfg.Transport
  }

diff --git a/provider/1pondo/core/core.go b/provider/1pondo/core/core.go
@@ -48,10 +48,14 @@ type Core struct {
 
 func (core *Core) Init() *Core {
  t := http.DefaultTransport.(*http.Transport).Clone()
- t.MaxIdleConnsPerHost = 1000
- t.IdleConnTimeout = 90 * time.Minute
+ t.MaxConnsPerHost = 5
+ t.MaxIdleConnsPerHost = 5
+ t.IdleConnTimeout = 5 * time.Minute
 
- core.Fetcher = fetch.Default(&fetch.Config{Transport: t})
+ core.Fetcher = fetch.Default(&fetch.Config{
+ Transport: t,
+ Timeout: 15 * time.Second,
+ })
  core.Scraper = scraper.NewDefaultScraper(core.DefaultName, core.BaseURL, core.DefaultPriority,
  scraper.WithHeaders(map[string]string{
  "Content-Type": "application/json",