From 585c3b8e649a14e224fe29d8a0de1ca94eb5f5b4 Mon Sep 17 00:00:00 2001 From: Jason Lyu Date: Sat, 30 Dec 2023 06:29:55 +0800 Subject: [PATCH] Feature(scraper): support FlareSolverr (#31) --- cmd/server/main.go | 6 +- common/flaresolverr/buffer.go | 26 +++++++ common/flaresolverr/client.go | 67 +++++++++++++++++++ common/flaresolverr/roundtripper.go | 14 ++++ engine/actor.go | 16 ++++- engine/engine.go | 53 +++++++++++++-- engine/movie.go | 22 ++++-- go.mod | 3 +- go.sum | 7 +- provider/1pondo/core/core.go | 3 +- provider/airav/airav.go | 3 +- provider/arzon/arzon.go | 3 +- provider/avbase/avbase.go | 3 +- provider/aventertainments/aventertainments.go | 3 +- provider/caribbeancom/core/core.go | 2 + provider/duga/duga.go | 3 +- provider/fanza/fanza.go | 3 +- provider/fc2/fc2.go | 3 +- provider/fc2hub/fc2hub.go | 3 +- provider/gcolle/gcolle.go | 6 +- provider/getchu/getchu.go | 3 +- provider/gfriends/gfriends.go | 3 + provider/h0930/core/core.go | 2 + provider/heydouga/heydouga.go | 3 +- provider/heyzo/heyzo.go | 3 +- provider/internal/scraper/option.go | 10 ++- provider/internal/scraper/scraper.go | 45 +++++++++++-- provider/jav321/jav321.go | 10 +-- provider/javbus/javbus.go | 3 +- provider/kin8tengoku/kin8tengoku.go | 3 +- provider/madouqu/madouqu.go | 11 ++- provider/mgstage/mgstage.go | 3 +- provider/mywife/mywife.go | 3 +- provider/pcolle/pcolle.go | 3 +- provider/prestige/prestige.go | 3 +- provider/provider.go | 10 +++ provider/sod/sod.go | 3 +- provider/tokyo-hot/tokyo-hot.go | 3 +- provider/xslist/xslist.go | 7 +- provider/xxx-av/xxx-av.go | 3 +- route/search.go | 5 +- 41 files changed, 326 insertions(+), 62 deletions(-) create mode 100644 common/flaresolverr/buffer.go create mode 100644 common/flaresolverr/client.go create mode 100644 common/flaresolverr/roundtripper.go diff --git a/cmd/server/main.go b/cmd/server/main.go index 52052ac2..7ce74876 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -34,7 +34,8 @@ type options struct { dsn string // engine options - requestTimeout time.Duration + requestTimeout time.Duration + flareSolverrURL string // database options dbMaxIdleConns int @@ -56,6 +57,7 @@ func init() { flag.StringVar(&opts.token, "token", "", "Token to access server") flag.StringVar(&opts.dsn, "dsn", "", "Database Service Name") flag.DurationVar(&opts.requestTimeout, "request-timeout", time.Minute, "Timeout per request") + flag.StringVar(&opts.flareSolverrURL, "flaresolverr-url", "", "FlareSolverr base url") flag.IntVar(&opts.dbMaxIdleConns, "db-max-idle-conns", 0, "Database max idle connections") flag.IntVar(&opts.dbMaxOpenConns, "db-max-open-conns", 0, "Database max open connections") flag.BoolVar(&opts.dbAutoMigrate, "db-auto-migrate", false, "Database auto migration") @@ -96,7 +98,7 @@ func main() { opts.requestTimeout = defaultRequestTimeout } - app := engine.New(db, opts.requestTimeout) + app := engine.New(db, opts.requestTimeout, opts.flareSolverrURL) if err = app.AutoMigrate(opts.dbAutoMigrate); err != nil { log.Fatal(err) } diff --git a/common/flaresolverr/buffer.go b/common/flaresolverr/buffer.go new file mode 100644 index 00000000..b52de2c3 --- /dev/null +++ b/common/flaresolverr/buffer.go @@ -0,0 +1,26 @@ +package flaresolverr + +import ( + "bytes" + "io" +) + +var _ io.ReadCloser = (*readCloser)(nil) + +type readCloser struct { + *bytes.Reader +} + +func newReadCloser(b []byte) *readCloser { + return &readCloser{ + Reader: bytes.NewReader(b), + } +} + +func newReadCloserString(s string) *readCloser { + return newReadCloser([]byte(s)) +} + +func (b *readCloser) Close() error { + return nil +} diff --git a/common/flaresolverr/client.go b/common/flaresolverr/client.go new file mode 100644 index 00000000..fdc430fb --- /dev/null +++ b/common/flaresolverr/client.go @@ -0,0 +1,67 @@ +package flaresolverr + +import ( + "fmt" + "net/http" + "net/url" + "time" + + gofs "github.com/SkYNewZ/go-flaresolverr" + "github.com/google/uuid" + "golang.org/x/net/context" +) + +type Client struct { + client gofs.Client + session uuid.UUID +} + +func New(url string, timeout time.Duration, session uuid.UUID) *Client { + return &Client{ + client: gofs.New(url, timeout, nil), + session: session, + } +} + +func (c *Client) Do(req *http.Request) (*http.Response, error) { + var ( + resp *gofs.Response + err error + ) + switch req.Method { + case http.MethodGet: + resp, err = c.client.Get(context.Background(), req.URL.String(), c.session) + case http.MethodPost: + fallthrough + default: + return nil, fmt.Errorf("unsupported method: %s", req.Method) + } + if err != nil { + return nil, err + } + + header := http.Header{} + header.Set("Content-Type", http.DetectContentType([]byte(resp.Solution.Response))) + + return &http.Response{ + Status: http.StatusText(http.StatusOK), + StatusCode: http.StatusOK, + Request: req, + Header: header, + Body: newReadCloserString(resp.Solution.Response), + }, nil +} + +func (c *Client) Get(rawURL string) (*http.Response, error) { + u, err := url.Parse(rawURL) + if err != nil { + return nil, err + } + return c.Do(&http.Request{Method: http.MethodGet, URL: u}) +} + +func (c *Client) StandardClient() *http.Client { + return &http.Client{ + Transport: &RoundTripper{Client: c}, + } +} diff --git a/common/flaresolverr/roundtripper.go b/common/flaresolverr/roundtripper.go new file mode 100644 index 00000000..bed484c5 --- /dev/null +++ b/common/flaresolverr/roundtripper.go @@ -0,0 +1,14 @@ +package flaresolverr + +import ( + "net/http" +) + +type RoundTripper struct { + Client *Client +} + +// RoundTrip satisfies the http.RoundTripper interface. +func (rt *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + return rt.Client.Do(req) +} diff --git a/engine/actor.go b/engine/actor.go index 5fb42f34..948d45f6 100644 --- a/engine/actor.go +++ b/engine/actor.go @@ -1,6 +1,7 @@ package engine import ( + goerrors "errors" "fmt" "sort" "sync" @@ -85,7 +86,7 @@ func (e *Engine) searchActor(keyword string, provider mt.Provider, fallback bool innerResults, innerErr := innerSearch(name) if innerErr != nil && // ignore InfoNotFound error. - innerErr != mt.ErrInfoNotFound { + !goerrors.Is(innerErr, mt.ErrInfoNotFound) { // add error to chain and handle it later. errors = append(errors, innerErr) continue @@ -109,12 +110,21 @@ func (e *Engine) SearchActor(keyword, name string, fallback bool) ([]*model.Acto return e.searchActor(keyword, provider, fallback) } -func (e *Engine) SearchActorAll(keyword string, fallback bool) (results []*model.ActorSearchResult, err error) { +func (e *Engine) SearchActorAll(keyword, lang string, fallback bool) (results []*model.ActorSearchResult, err error) { var ( mu sync.Mutex wg sync.WaitGroup ) - for _, provider := range e.actorProviders { + + availableProviders := e.actorProviders + if lang != "" { + if availableProviders, err = e.GetActorProvidersByLanguage(lang); err != nil { + return + } + e.logger.Infof("Actor Keyword: %s, Language: %s, Providers: %v", keyword, lang, availableProviders) + } + + for _, provider := range availableProviders { wg.Add(1) go func(provider mt.ActorProvider) { defer wg.Done() diff --git a/engine/engine.go b/engine/engine.go index 4c038a77..7e93281b 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -7,6 +7,7 @@ import ( "time" "go.uber.org/zap" + "golang.org/x/text/language" "gorm.io/gorm" "github.com/metatube-community/metatube-sdk-go/common/fetch" @@ -28,15 +29,15 @@ type Engine struct { movieHostProviders map[string][]mt.MovieProvider } -func New(db *gorm.DB, timeout time.Duration) *Engine { +func New(db *gorm.DB, timeout time.Duration, flareSolverrURL string) *Engine { engine := &Engine{ db: db, fetcher: fetch.Default(nil), } logger, _ := zap.NewProduction() engine.logger = logger.Sugar() - engine.initActorProviders(timeout) - engine.initMovieProviders(timeout) + engine.initActorProviders(timeout, flareSolverrURL) + engine.initMovieProviders(timeout, flareSolverrURL) return engine } @@ -45,13 +46,13 @@ func Default() *Engine { DSN: "", DisableAutomaticPing: true, }) - engine := New(db, time.Minute) + engine := New(db, time.Minute, "") defer engine.AutoMigrate(true) return engine } // initActorProviders initializes actor providers. -func (e *Engine) initActorProviders(timeout time.Duration) { +func (e *Engine) initActorProviders(timeout time.Duration, flareSolverrURL string) { { // init e.actorProviders = make(map[string]mt.ActorProvider) e.actorHostProviders = make(map[string][]mt.ActorProvider) @@ -61,6 +62,9 @@ func (e *Engine) initActorProviders(timeout time.Duration) { if s, ok := provider.(mt.RequestTimeoutSetter); ok { s.SetRequestTimeout(timeout) } + if s, ok := provider.(mt.FlareSolverrSetter); ok { + s.SetFlareSolverr(flareSolverrURL) + } // Add actor provider by name. e.actorProviders[strings.ToUpper(name)] = provider // Add actor provider by host. @@ -70,7 +74,7 @@ func (e *Engine) initActorProviders(timeout time.Duration) { } // initMovieProviders initializes movie providers. -func (e *Engine) initMovieProviders(timeout time.Duration) { +func (e *Engine) initMovieProviders(timeout time.Duration, flareSolverrURL string) { { // init e.movieProviders = make(map[string]mt.MovieProvider) e.movieHostProviders = make(map[string][]mt.MovieProvider) @@ -80,6 +84,9 @@ func (e *Engine) initMovieProviders(timeout time.Duration) { if s, ok := provider.(mt.RequestTimeoutSetter); ok { s.SetRequestTimeout(timeout) } + if s, ok := provider.(mt.FlareSolverrSetter); ok { + s.SetFlareSolverr(flareSolverrURL) + } // Add movie provider by name. e.movieProviders[strings.ToUpper(name)] = provider // Add movie provider by host. @@ -97,6 +104,23 @@ func (e *Engine) GetActorProviders() map[string]mt.ActorProvider { return e.actorProviders } +func (e *Engine) GetActorProvidersByLanguage(lang string) (map[string]mt.ActorProvider, error) { + tag, err := language.Parse(lang) + if err != nil { + return nil, err + } + + providers := make(map[string]mt.ActorProvider) + matcher := language.NewMatcher([]language.Tag{tag}) + + for _, provider := range e.actorProviders { + if _, _, c := matcher.Match(provider.Language()); c >= language.Low { + providers[strings.ToUpper(provider.Name())] = provider + } + } + return providers, nil +} + func (e *Engine) GetActorProviderByURL(rawURL string) (mt.ActorProvider, error) { u, err := url.Parse(rawURL) if err != nil { @@ -135,6 +159,23 @@ func (e *Engine) GetMovieProviders() map[string]mt.MovieProvider { return e.movieProviders } +func (e *Engine) GetMovieProvidersByLanguage(lang string) (map[string]mt.MovieProvider, error) { + tag, err := language.Parse(lang) + if err != nil { + return nil, err + } + + providers := make(map[string]mt.MovieProvider) + matcher := language.NewMatcher([]language.Tag{tag}) + + for _, provider := range e.movieProviders { + if _, _, c := matcher.Match(provider.Language()); c >= language.Low { + providers[strings.ToUpper(provider.Name())] = provider + } + } + return providers, nil +} + func (e *Engine) GetMovieProviderByURL(rawURL string) (mt.MovieProvider, error) { u, err := url.Parse(rawURL) if err != nil { diff --git a/engine/movie.go b/engine/movie.go index 5327c7ff..0f483e9e 100644 --- a/engine/movie.go +++ b/engine/movie.go @@ -85,7 +85,7 @@ func (e *Engine) SearchMovie(keyword, name string, fallback bool) ([]*model.Movi return e.searchMovie(keyword, provider, fallback) } -func (e *Engine) searchMovieAll(keyword string) (results []*model.MovieSearchResult, err error) { +func (e *Engine) searchMovieAll(keyword string, providers map[string]mt.MovieProvider) (results []*model.MovieSearchResult, err error) { type response struct { Results []*model.MovieSearchResult Error error @@ -96,7 +96,7 @@ func (e *Engine) searchMovieAll(keyword string) (results []*model.MovieSearchRes respCh := make(chan response) var wg sync.WaitGroup - for _, provider := range e.movieProviders { + for _, provider := range providers { wg.Add(1) // Goroutine started time. startTime := time.Now() @@ -139,11 +139,19 @@ func (e *Engine) searchMovieAll(keyword string) (results []*model.MovieSearchRes } // SearchMovieAll searches the keyword from all providers. -func (e *Engine) SearchMovieAll(keyword string, fallback bool) (results []*model.MovieSearchResult, err error) { +func (e *Engine) SearchMovieAll(keyword, lang string, fallback bool) (results []*model.MovieSearchResult, err error) { if keyword = number.Trim(keyword); keyword == "" { return nil, mt.ErrInvalidKeyword } + availableProviders := e.movieProviders + if lang != "" { + if availableProviders, err = e.GetMovieProvidersByLanguage(lang); err != nil { + return + } + e.logger.Infof("Movie Keyword: %s, Language: %s, Providers: %v", keyword, lang, availableProviders) + } + defer func() { if err != nil { return @@ -180,12 +188,16 @@ func (e *Engine) SearchMovieAll(keyword string, fallback bool) (results []*model // overwrite error. err = nil // append results. - results = append(results, innerResults...) + for _, result := range innerResults { + if _, ok := availableProviders[strings.ToUpper(result.Provider)]; ok { + results = append(results, result) + } + } } }() } - results, err = e.searchMovieAll(keyword) + results, err = e.searchMovieAll(keyword, availableProviders) return } diff --git a/go.mod b/go.mod index 629c1700..6010f3f8 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/metatube-community/metatube-sdk-go go 1.21 require ( + github.com/SkYNewZ/go-flaresolverr v0.0.0-20230503120808-3826fdd382dd github.com/adrg/strutil v0.3.1 github.com/antchfx/htmlquery v1.3.0 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de @@ -11,6 +12,7 @@ require ( github.com/gin-gonic/gin v1.9.1 github.com/glebarez/sqlite v1.10.0 github.com/gocolly/colly/v2 v2.1.1-0.20230620150846-a6e3d81fe6b7 + github.com/google/uuid v1.5.0 github.com/grafov/m3u8 v0.12.0 github.com/hashicorp/go-retryablehttp v0.7.5 github.com/iancoleman/orderedmap v0.3.0 @@ -54,7 +56,6 @@ require ( github.com/goccy/go-json v0.10.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/uuid v1.5.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect diff --git a/go.sum b/go.sum index bdb11edd..01b96696 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= +github.com/SkYNewZ/go-flaresolverr v0.0.0-20230503120808-3826fdd382dd h1:dRFqhAE/LTVh/yyodUpMsLEcF6UixqjuiRX5eOuwTQQ= +github.com/SkYNewZ/go-flaresolverr v0.0.0-20230503120808-3826fdd382dd/go.mod h1:/Z29vstAqNLpmsGpJ1CedxtSHand1HiGtwLErRXi6mk= github.com/adrg/strutil v0.3.1 h1:OLvSS7CSJO8lBii4YmBt8jiK9QOtB9CzCzwl4Ic/Fz4= github.com/adrg/strutil v0.3.1/go.mod h1:8h90y18QLrs11IBffcGX3NW/GFBXCMcNg4M7H6MspPA= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= @@ -108,8 +110,9 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= @@ -320,8 +323,6 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= diff --git a/provider/1pondo/core/core.go b/provider/1pondo/core/core.go index 31c6fe37..4941e95b 100644 --- a/provider/1pondo/core/core.go +++ b/provider/1pondo/core/core.go @@ -12,6 +12,7 @@ import ( "github.com/gocolly/colly/v2" "github.com/nlnwa/whatwg-url/url" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/parser" "github.com/metatube-community/metatube-sdk-go/model" @@ -45,7 +46,7 @@ type Core struct { } func (core *Core) Init() *Core { - core.Scraper = scraper.NewDefaultScraper(core.DefaultName, core.BaseURL, core.DefaultPriority, + core.Scraper = scraper.NewDefaultScraper(core.DefaultName, core.BaseURL, core.DefaultPriority, language.Japanese, scraper.WithHeaders(map[string]string{ "Content-Type": "application/json", }), diff --git a/provider/airav/airav.go b/provider/airav/airav.go index ae852567..2b527c6f 100644 --- a/provider/airav/airav.go +++ b/provider/airav/airav.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/gocolly/colly/v2" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/parser" "github.com/metatube-community/metatube-sdk-go/model" @@ -39,7 +40,7 @@ type AirAV struct { func New() *AirAV { return &AirAV{ - Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, + Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese, scraper.WithHeaders(map[string]string{ "Origin": baseURL, "Referer": baseURL, diff --git a/provider/arzon/arzon.go b/provider/arzon/arzon.go index f5b8a2b1..493b5bf5 100644 --- a/provider/arzon/arzon.go +++ b/provider/arzon/arzon.go @@ -9,6 +9,7 @@ import ( "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/fetch" "github.com/metatube-community/metatube-sdk-go/common/number" @@ -44,7 +45,7 @@ type ARZON struct { func New() *ARZON { return &ARZON{ Fetcher: fetch.Default(&fetch.Config{Referer: baseURL}), - Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority), + Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese), } } diff --git a/provider/avbase/avbase.go b/provider/avbase/avbase.go index 9cbcb05b..a366c5c2 100644 --- a/provider/avbase/avbase.go +++ b/provider/avbase/avbase.go @@ -11,6 +11,7 @@ import ( "time" "github.com/gocolly/colly/v2" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/number" "github.com/metatube-community/metatube-sdk-go/common/parser" @@ -50,7 +51,7 @@ type AVBase struct { func New() *AVBase { return &AVBase{ - Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, + Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese, scraper.WithHeaders(map[string]string{ "Referer": baseURL, })), diff --git a/provider/aventertainments/aventertainments.go b/provider/aventertainments/aventertainments.go index ef314d6d..9e8e22cf 100644 --- a/provider/aventertainments/aventertainments.go +++ b/provider/aventertainments/aventertainments.go @@ -9,6 +9,7 @@ import ( "github.com/antchfx/htmlquery" "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/number" "github.com/metatube-community/metatube-sdk-go/common/parser" @@ -38,7 +39,7 @@ type AVE struct { } func New() *AVE { - return &AVE{scraper.NewDefaultScraper(Name, baseURL, Priority)} + return &AVE{scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese)} } func (ave *AVE) GetMovieInfoByID(id string) (info *model.MovieInfo, err error) { diff --git a/provider/caribbeancom/core/core.go b/provider/caribbeancom/core/core.go index 226d632d..cf3bdaef 100644 --- a/provider/caribbeancom/core/core.go +++ b/provider/caribbeancom/core/core.go @@ -12,6 +12,7 @@ import ( "github.com/antchfx/htmlquery" "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/parser" "github.com/metatube-community/metatube-sdk-go/model" @@ -36,6 +37,7 @@ func (core *Core) Init() *Core { core.DefaultName, core.BaseURL, core.DefaultPriority, + language.Japanese, scraper.WithDetectCharset()) return core } diff --git a/provider/duga/duga.go b/provider/duga/duga.go index aa412cec..bb626661 100644 --- a/provider/duga/duga.go +++ b/provider/duga/duga.go @@ -10,6 +10,7 @@ import ( "time" "github.com/gocolly/colly/v2" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/number" "github.com/metatube-community/metatube-sdk-go/common/parser" @@ -40,7 +41,7 @@ type DUGA struct { func New() *DUGA { return &DUGA{ - Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority), + Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese), } } diff --git a/provider/fanza/fanza.go b/provider/fanza/fanza.go index 16c93ff8..b796efe6 100644 --- a/provider/fanza/fanza.go +++ b/provider/fanza/fanza.go @@ -17,6 +17,7 @@ import ( "github.com/gocolly/colly/v2" "github.com/iancoleman/orderedmap" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/comparer" "github.com/metatube-community/metatube-sdk-go/common/number" @@ -57,7 +58,7 @@ type FANZA struct { func New() *FANZA { return &FANZA{ - Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, + Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese, scraper.WithCookies(baseURL, []*http.Cookie{ {Name: "age_check_done", Value: "1"}, })), diff --git a/provider/fc2/fc2.go b/provider/fc2/fc2.go index 5fd89bdf..fddd3c59 100644 --- a/provider/fc2/fc2.go +++ b/provider/fc2/fc2.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/gocolly/colly/v2" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/parser" "github.com/metatube-community/metatube-sdk-go/model" @@ -33,7 +34,7 @@ type FC2 struct { } func New() *FC2 { - return &FC2{scraper.NewDefaultScraper(Name, baseURL, Priority)} + return &FC2{scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese)} } func (fc2 *FC2) NormalizeMovieID(id string) string { diff --git a/provider/fc2hub/fc2hub.go b/provider/fc2hub/fc2hub.go index 25f6043c..b9e9621c 100644 --- a/provider/fc2hub/fc2hub.go +++ b/provider/fc2hub/fc2hub.go @@ -10,6 +10,7 @@ import ( "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/parser" "github.com/metatube-community/metatube-sdk-go/model" @@ -39,7 +40,7 @@ type FC2HUB struct { } func New() *FC2HUB { - return &FC2HUB{scraper.NewDefaultScraper(Name, baseURL, Priority)} + return &FC2HUB{scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese)} } func (fc2hub *FC2HUB) GetMovieInfoByID(id string) (info *model.MovieInfo, err error) { diff --git a/provider/gcolle/gcolle.go b/provider/gcolle/gcolle.go index bf73c593..61579a6f 100644 --- a/provider/gcolle/gcolle.go +++ b/provider/gcolle/gcolle.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/gocolly/colly/v2" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/parser" "github.com/metatube-community/metatube-sdk-go/model" @@ -34,7 +35,10 @@ type Gcolle struct { } func New() *Gcolle { - return &Gcolle{scraper.NewDefaultScraper(Name, baseURL, Priority, scraper.WithDetectCharset())} + return &Gcolle{scraper.NewDefaultScraper( + Name, baseURL, Priority, language.Japanese, + scraper.WithDetectCharset(), + )} } func (gcl *Gcolle) NormalizeMovieID(id string) string { diff --git a/provider/getchu/getchu.go b/provider/getchu/getchu.go index 0a5fcc01..9ac38b8a 100644 --- a/provider/getchu/getchu.go +++ b/provider/getchu/getchu.go @@ -10,6 +10,7 @@ import ( "github.com/antchfx/htmlquery" "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/parser" "github.com/metatube-community/metatube-sdk-go/model" @@ -34,7 +35,7 @@ type Getchu struct { } func New() *Getchu { - return &Getchu{scraper.NewDefaultScraper(Name, baseURL, Priority)} + return &Getchu{scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese)} } func (gcu *Getchu) NormalizeMovieID(id string) string { diff --git a/provider/gfriends/gfriends.go b/provider/gfriends/gfriends.go index c100f214..af3732f1 100644 --- a/provider/gfriends/gfriends.go +++ b/provider/gfriends/gfriends.go @@ -8,6 +8,7 @@ import ( "time" "github.com/iancoleman/orderedmap" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/fetch" "github.com/metatube-community/metatube-sdk-go/common/reverse" @@ -47,6 +48,8 @@ func (gf *GFriends) Name() string { return Name } func (gf *GFriends) Priority() int { return Priority } +func (gf *GFriends) Language() language.Tag { return language.Japanese } + func (gf *GFriends) URL() *url.URL { return _baseURL } func (gf *GFriends) NormalizeActorID(id string) string { return id /* AS IS */ } diff --git a/provider/h0930/core/core.go b/provider/h0930/core/core.go index 0434d913..68751d68 100644 --- a/provider/h0930/core/core.go +++ b/provider/h0930/core/core.go @@ -10,6 +10,7 @@ import ( "time" "github.com/gocolly/colly/v2" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/parser" "github.com/metatube-community/metatube-sdk-go/model" @@ -34,6 +35,7 @@ func (core *Core) Init() *Core { core.DefaultName, core.BaseURL, core.DefaultPriority, + language.Japanese, scraper.WithDetectCharset()) return core } diff --git a/provider/heydouga/heydouga.go b/provider/heydouga/heydouga.go index 769a5459..4e6b61b1 100644 --- a/provider/heydouga/heydouga.go +++ b/provider/heydouga/heydouga.go @@ -10,6 +10,7 @@ import ( "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/parser" "github.com/metatube-community/metatube-sdk-go/model" @@ -35,7 +36,7 @@ type HeyDouga struct { } func New() *HeyDouga { - return &HeyDouga{scraper.NewDefaultScraper(Name, baseURL, Priority)} + return &HeyDouga{scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese)} } func (hey *HeyDouga) NormalizeMovieID(id string) string { diff --git a/provider/heyzo/heyzo.go b/provider/heyzo/heyzo.go index 5620d078..0def0090 100644 --- a/provider/heyzo/heyzo.go +++ b/provider/heyzo/heyzo.go @@ -11,6 +11,7 @@ import ( "strings" "github.com/gocolly/colly/v2" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/js" "github.com/metatube-community/metatube-sdk-go/common/m3u8" @@ -43,7 +44,7 @@ type Heyzo struct { } func New() *Heyzo { - return &Heyzo{scraper.NewDefaultScraper(Name, baseURL, Priority)} + return &Heyzo{scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese)} } func (hzo *Heyzo) GetMovieReviewsByID(id string) (reviews []*model.MovieReviewDetail, err error) { diff --git a/provider/internal/scraper/option.go b/provider/internal/scraper/option.go index 8327d521..996e8fc7 100644 --- a/provider/internal/scraper/option.go +++ b/provider/internal/scraper/option.go @@ -64,8 +64,9 @@ func WithCookies(url string, cookies []*http.Cookie) Option { } } -func WithRequestTimeout(timeout time.Duration) Option { +func WithForcedRequestTimeout(timeout time.Duration) Option { return func(s *Scraper) error { + s.timeout = timeout s.c.SetRequestTimeout(timeout) return nil } @@ -95,6 +96,13 @@ func WithTransport(transport http.RoundTripper) Option { } } +func WithEnableFlareSolverr() Option { + return func(s *Scraper) error { + s.flareSolverrEnabled = true + return nil + } +} + func WithLimit(rule *colly.LimitRule) Option { return func(s *Scraper) error { return s.c.Limit(rule) diff --git a/provider/internal/scraper/scraper.go b/provider/internal/scraper/scraper.go index a8cd65ec..1b519f67 100644 --- a/provider/internal/scraper/scraper.go +++ b/provider/internal/scraper/scraper.go @@ -5,22 +5,36 @@ import ( "time" "github.com/gocolly/colly/v2" + "github.com/google/uuid" + "golang.org/x/text/language" + "github.com/metatube-community/metatube-sdk-go/common/flaresolverr" "github.com/metatube-community/metatube-sdk-go/provider" ) -var _ provider.Provider = (*Scraper)(nil) +var ( + _ provider.Provider = (*Scraper)(nil) + _ provider.RequestTimeoutSetter = (*Scraper)(nil) + _ provider.FlareSolverrSetter = (*Scraper)(nil) +) // Scraper implements basic Provider interface. type Scraper struct { name string priority int + lang language.Tag baseURL *url.URL c *colly.Collector + + // Timeout + timeout time.Duration + + // FlareSolverr + flareSolverrEnabled bool } // NewScraper returns Provider implemented *Scraper. -func NewScraper(name, baseURL string, priority int, opts ...Option) *Scraper { +func NewScraper(name, baseURL string, priority int, lang language.Tag, opts ...Option) *Scraper { u, err := url.Parse(baseURL) if err != nil { panic(err) @@ -28,6 +42,7 @@ func NewScraper(name, baseURL string, priority int, opts ...Option) *Scraper { s := &Scraper{ name: name, priority: priority, + lang: lang, baseURL: u, c: colly.NewCollector(), } @@ -41,8 +56,8 @@ func NewScraper(name, baseURL string, priority int, opts ...Option) *Scraper { } // NewDefaultScraper returns a *Scraper with default options enabled. -func NewDefaultScraper(name, baseURL string, priority int, opts ...Option) *Scraper { - return NewScraper(name, baseURL, priority, append([]Option{ +func NewDefaultScraper(name, baseURL string, priority int, lang language.Tag, opts ...Option) *Scraper { + return NewScraper(name, baseURL, priority, lang, append([]Option{ WithAllowURLRevisit(), WithIgnoreRobotsTxt(), WithRandomUserAgent(), @@ -55,6 +70,8 @@ func (s *Scraper) URL() *url.URL { return s.baseURL } func (s *Scraper) Priority() int { return s.priority } +func (s *Scraper) Language() language.Tag { return s.lang } + func (s *Scraper) NormalizeMovieID(id string) string { return id /* AS IS */ } func (s *Scraper) ParseMovieIDFromURL(string) (string, error) { panic("unimplemented") } @@ -67,4 +84,22 @@ func (s *Scraper) ParseActorIDFromURL(string) (string, error) { panic("unimpleme func (s *Scraper) ClonedCollector() *colly.Collector { return s.c.Clone() } // SetRequestTimeout sets timeout for HTTP requests. -func (s *Scraper) SetRequestTimeout(timeout time.Duration) { s.c.SetRequestTimeout(timeout) } +func (s *Scraper) SetRequestTimeout(timeout time.Duration) { + if s.timeout == 0 { + s.timeout = timeout + } + s.c.SetRequestTimeout(s.timeout) +} + +// SetFlareSolverr initiates the flaresolverr client for HTTP requests. +func (s *Scraper) SetFlareSolverr(baseURL string) { + if s.flareSolverrEnabled && baseURL != "" { + u, err := url.Parse(baseURL) + if err != nil { + panic(err) + } + s.c.WithTransport(&flaresolverr.RoundTripper{ + Client: flaresolverr.New(u.String(), s.timeout, uuid.NameSpaceURL /* as Session Key */), + }) + } +} diff --git a/provider/jav321/jav321.go b/provider/jav321/jav321.go index f24d3150..6252df2b 100644 --- a/provider/jav321/jav321.go +++ b/provider/jav321/jav321.go @@ -11,6 +11,7 @@ import ( "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/number" "github.com/metatube-community/metatube-sdk-go/common/parser" @@ -40,11 +41,10 @@ type JAV321 struct { } func New() *JAV321 { - return &JAV321{scraper.NewDefaultScraper(Name, baseURL, Priority)} -} - -func (jav *JAV321) SetRequestTimeout(_ time.Duration) { - jav.Scraper.SetRequestTimeout(10 * time.Second) + return &JAV321{scraper.NewDefaultScraper( + Name, baseURL, Priority, language.Japanese, + scraper.WithForcedRequestTimeout(10*time.Second), + )} } func (jav *JAV321) GetMovieInfoByID(id string) (info *model.MovieInfo, err error) { diff --git a/provider/javbus/javbus.go b/provider/javbus/javbus.go index de760afb..dbbb342a 100644 --- a/provider/javbus/javbus.go +++ b/provider/javbus/javbus.go @@ -10,6 +10,7 @@ import ( "sync" "github.com/gocolly/colly/v2" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/number" "github.com/metatube-community/metatube-sdk-go/common/parser" @@ -41,7 +42,7 @@ type JavBus struct { func New() *JavBus { return &JavBus{ - Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, + Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese, scraper.WithDisableRedirects(), scraper.WithCookies(baseURL, []*http.Cookie{ // existmag=all diff --git a/provider/kin8tengoku/kin8tengoku.go b/provider/kin8tengoku/kin8tengoku.go index 0aba8007..9bc2abca 100644 --- a/provider/kin8tengoku/kin8tengoku.go +++ b/provider/kin8tengoku/kin8tengoku.go @@ -13,6 +13,7 @@ import ( "github.com/antchfx/htmlquery" "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/parser" "github.com/metatube-community/metatube-sdk-go/model" @@ -38,7 +39,7 @@ type KIN8 struct { } func New() *KIN8 { - return &KIN8{scraper.NewDefaultScraper(Name, baseURL, Priority)} + return &KIN8{scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese)} } func (k8 *KIN8) NormalizeMovieID(id string) string { diff --git a/provider/madouqu/madouqu.go b/provider/madouqu/madouqu.go index 057e9227..dde586ee 100644 --- a/provider/madouqu/madouqu.go +++ b/provider/madouqu/madouqu.go @@ -6,9 +6,9 @@ import ( "path" "regexp" "strings" - "time" "github.com/gocolly/colly/v2" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/number" "github.com/metatube-community/metatube-sdk-go/common/parser" @@ -38,11 +38,10 @@ type MadouQu struct { } func New() *MadouQu { - return &MadouQu{scraper.NewDefaultScraper(Name, baseURL, Priority)} -} - -func (mdq *MadouQu) SetRequestTimeout(_ time.Duration) { - mdq.Scraper.SetRequestTimeout(10 * time.Second) // force timeout setting. + return &MadouQu{scraper.NewDefaultScraper( + Name, baseURL, Priority, language.Chinese, + scraper.WithEnableFlareSolverr(), + )} } func (mdq *MadouQu) GetMovieInfoByID(id string) (info *model.MovieInfo, err error) { diff --git a/provider/mgstage/mgstage.go b/provider/mgstage/mgstage.go index a7f83090..013151c7 100644 --- a/provider/mgstage/mgstage.go +++ b/provider/mgstage/mgstage.go @@ -13,6 +13,7 @@ import ( "github.com/antchfx/htmlquery" "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/number" "github.com/metatube-community/metatube-sdk-go/common/parser" @@ -45,7 +46,7 @@ type MGS struct { func New() *MGS { return &MGS{ - Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, + Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese, scraper.WithCookies(baseURL, []*http.Cookie{ {Name: "adc", Value: "1"}, })), diff --git a/provider/mywife/mywife.go b/provider/mywife/mywife.go index df453971..3afd4d6d 100644 --- a/provider/mywife/mywife.go +++ b/provider/mywife/mywife.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/gocolly/colly/v2" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/model" "github.com/metatube-community/metatube-sdk-go/provider" @@ -31,7 +32,7 @@ type MyWife struct { } func New() *MyWife { - return &MyWife{scraper.NewDefaultScraper(Name, baseURL, Priority)} + return &MyWife{scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese)} } func (mw *MyWife) NormalizeMovieID(id string) string { diff --git a/provider/pcolle/pcolle.go b/provider/pcolle/pcolle.go index e687eab7..6b3aa1d5 100644 --- a/provider/pcolle/pcolle.go +++ b/provider/pcolle/pcolle.go @@ -10,6 +10,7 @@ import ( "github.com/antchfx/htmlquery" "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/parser" "github.com/metatube-community/metatube-sdk-go/model" @@ -35,7 +36,7 @@ type Pcolle struct { func New() *Pcolle { return &Pcolle{ - Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, + Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese, scraper.WithCookies(baseURL, []*http.Cookie{ {Name: "AGE_CONF", Value: "1"}, })), diff --git a/provider/prestige/prestige.go b/provider/prestige/prestige.go index 4df96543..e396863b 100644 --- a/provider/prestige/prestige.go +++ b/provider/prestige/prestige.go @@ -11,6 +11,7 @@ import ( "github.com/antchfx/htmlquery" "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/number" "github.com/metatube-community/metatube-sdk-go/common/parser" @@ -41,7 +42,7 @@ type PRESTIGE struct { func New() *PRESTIGE { return &PRESTIGE{ - Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, + Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese, scraper.WithCookies(baseURL, []*http.Cookie{ {Name: "coc", Value: "1"}, {Name: "age_auth", Value: "1"}, diff --git a/provider/provider.go b/provider/provider.go index 604d9bc4..6fcc2121 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -5,6 +5,8 @@ import ( "net/url" "time" + "golang.org/x/text/language" + "github.com/metatube-community/metatube-sdk-go/model" ) @@ -12,6 +14,9 @@ type Provider interface { // Name returns the name of the provider. Name() string + // Language returns the default language of the provider. + Language() language.Tag + // Priority returns the matching priority of the provider. Priority() int @@ -83,3 +88,8 @@ type RequestTimeoutSetter interface { // SetRequestTimeout sets timeout for HTTP requests. SetRequestTimeout(timeout time.Duration) } + +type FlareSolverrSetter interface { + // SetFlareSolverr initiates the flaresolverr client for HTTP requests. + SetFlareSolverr(baseURL string) +} diff --git a/provider/sod/sod.go b/provider/sod/sod.go index e69903fe..a585e4e0 100644 --- a/provider/sod/sod.go +++ b/provider/sod/sod.go @@ -8,6 +8,7 @@ import ( "github.com/antchfx/htmlquery" "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/fetch" "github.com/metatube-community/metatube-sdk-go/common/number" @@ -44,7 +45,7 @@ type SOD struct { func New() *SOD { return &SOD{ Fetcher: fetch.Default(&fetch.Config{Referer: baseURL}), - Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority), + Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese), } } diff --git a/provider/tokyo-hot/tokyo-hot.go b/provider/tokyo-hot/tokyo-hot.go index 1997d1a9..a29ddf6f 100644 --- a/provider/tokyo-hot/tokyo-hot.go +++ b/provider/tokyo-hot/tokyo-hot.go @@ -12,6 +12,7 @@ import ( "github.com/antchfx/htmlquery" "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/parser" "github.com/metatube-community/metatube-sdk-go/model" @@ -40,7 +41,7 @@ type TokyoHot struct { } func New() *TokyoHot { - return &TokyoHot{scraper.NewDefaultScraper(Name, baseURL, Priority, + return &TokyoHot{scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese, // Temporary workaround for unknown CA issue. scraper.WithTransport(&http.Transport{ Proxy: http.DefaultTransport.(*http.Transport).Proxy, diff --git a/provider/xslist/xslist.go b/provider/xslist/xslist.go index ad1f5a5e..2bce2a5e 100644 --- a/provider/xslist/xslist.go +++ b/provider/xslist/xslist.go @@ -10,6 +10,7 @@ import ( "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" dt "gorm.io/datatypes" "github.com/metatube-community/metatube-sdk-go/common/parser" @@ -39,7 +40,11 @@ type XsList struct { } func New() *XsList { - return &XsList{scraper.NewDefaultScraper(Name, baseURL, Priority, scraper.WithDisableCookies())} + return &XsList{scraper.NewDefaultScraper( + Name, baseURL, Priority, language.Japanese, + scraper.WithDisableCookies(), + scraper.WithEnableFlareSolverr(), + )} } func (xsl *XsList) GetActorInfoByID(id string) (info *model.ActorInfo, err error) { diff --git a/provider/xxx-av/xxx-av.go b/provider/xxx-av/xxx-av.go index 1a4ea5ea..6e566913 100644 --- a/provider/xxx-av/xxx-av.go +++ b/provider/xxx-av/xxx-av.go @@ -11,6 +11,7 @@ import ( "github.com/antchfx/htmlquery" "github.com/gocolly/colly/v2" "golang.org/x/net/html" + "golang.org/x/text/language" "github.com/metatube-community/metatube-sdk-go/common/parser" "github.com/metatube-community/metatube-sdk-go/model" @@ -36,7 +37,7 @@ type TripleX struct { func New() *TripleX { return &TripleX{ - Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, + Scraper: scraper.NewDefaultScraper(Name, baseURL, Priority, language.Japanese, scraper.WithCookies(baseURL, []*http.Cookie{ {Name: "acc_accept_lang", Value: "japanese"}, })), diff --git a/route/search.go b/route/search.go index d6b285c4..0c727960 100644 --- a/route/search.go +++ b/route/search.go @@ -21,6 +21,7 @@ const ( type searchQuery struct { Q string `form:"q" binding:"required"` Provider string `form:"provider"` + Language string `form:"lang"` Fallback bool `form:"fallback"` } @@ -53,7 +54,7 @@ func getSearch(app *engine.Engine, typ searchType) gin.HandlerFunc { if isValidURL { results, err = app.GetActorInfoByURL(query.Q, true /* always lazy */) } else if searchAll { - results, err = app.SearchActorAll(query.Q, query.Fallback) + results, err = app.SearchActorAll(query.Q, query.Language, query.Fallback) } else { results, err = app.SearchActor(query.Q, query.Provider, query.Fallback) } @@ -61,7 +62,7 @@ func getSearch(app *engine.Engine, typ searchType) gin.HandlerFunc { if isValidURL { results, err = app.GetMovieInfoByURL(query.Q, true /* always lazy */) } else if searchAll { - results, err = app.SearchMovieAll(query.Q, query.Fallback) + results, err = app.SearchMovieAll(query.Q, query.Language, query.Fallback) } else { results, err = app.SearchMovie(query.Q, query.Provider, query.Fallback) }