Revert(scraper): revert support for FlareSolverr

This reverts commit 585c3b8.

cmd/server/main.go

@@ -34,8 +34,7 @@ type options struct {
  dsn string
 
  // engine options
- requestTimeout time.Duration
- flareSolverrURL string
+ requestTimeout time.Duration
 
  // database options
  dbMaxIdleConns int
@@ -57,7 +56,6 @@ func init() {
  flag.StringVar(&opts.token, "token", "", "Token to access server")
  flag.StringVar(&opts.dsn, "dsn", "", "Database Service Name")
  flag.DurationVar(&opts.requestTimeout, "request-timeout", time.Minute, "Timeout per request")
- flag.StringVar(&opts.flareSolverrURL, "flaresolverr-url", "", "FlareSolverr base url")
  flag.IntVar(&opts.dbMaxIdleConns, "db-max-idle-conns", 0, "Database max idle connections")
  flag.IntVar(&opts.dbMaxOpenConns, "db-max-open-conns", 0, "Database max open connections")
  flag.BoolVar(&opts.dbAutoMigrate, "db-auto-migrate", false, "Database auto migration")
@@ -98,7 +96,7 @@ func main() {
  opts.requestTimeout = defaultRequestTimeout
  }
 
- app := engine.New(db, opts.requestTimeout, opts.flareSolverrURL)
+ app := engine.New(db, opts.requestTimeout)
  if err = app.AutoMigrate(opts.dbAutoMigrate); err != nil {
  log.Fatal(err)
  }

engine/actor.go

@@ -1,7 +1,6 @@
 package engine
 
 import (
- goerrors "errors"
  "fmt"
  "sort"
  "sync"
@@ -86,7 +85,7 @@ func (e *Engine) searchActor(keyword string, provider mt.Provider, fallback bool
  innerResults, innerErr := innerSearch(name)
  if innerErr != nil &&
  // ignore InfoNotFound error.
- !goerrors.Is(innerErr, mt.ErrInfoNotFound) {
+ innerErr != mt.ErrInfoNotFound {
  // add error to chain and handle it later.
  errors = append(errors, innerErr)
  continue
@@ -110,21 +109,12 @@ func (e *Engine) SearchActor(keyword, name string, fallback bool) ([]*model.Acto
  return e.searchActor(keyword, provider, fallback)
 }
 
-func (e *Engine) SearchActorAll(keyword, lang string, fallback bool) (results []*model.ActorSearchResult, err error) {
+func (e *Engine) SearchActorAll(keyword string, fallback bool) (results []*model.ActorSearchResult, err error) {
  var (
  mu sync.Mutex
  wg sync.WaitGroup
  )
-
- availableProviders := e.actorProviders
- if lang != "" {
- if availableProviders, err = e.GetActorProvidersByLanguage(lang); err != nil {
- return
- }
- e.logger.Infof("Actor Keyword: %s, Language: %s, Providers: %v", keyword, lang, availableProviders)
- }
-
- for _, provider := range availableProviders {
+ for _, provider := range e.actorProviders {
  wg.Add(1)
  go func(provider mt.ActorProvider) {
  defer wg.Done()

engine/engine.go

@@ -7,7 +7,6 @@ import (
  "time"
 
  "go.uber.org/zap"
- "golang.org/x/text/language"
  "gorm.io/gorm"
 
  "github.com/metatube-community/metatube-sdk-go/common/fetch"
@@ -29,15 +28,15 @@ type Engine struct {
  movieHostProviders map[string][]mt.MovieProvider
 }
 
-func New(db *gorm.DB, timeout time.Duration, flareSolverrURL string) *Engine {
+func New(db *gorm.DB, timeout time.Duration) *Engine {
  engine := &Engine{
  db: db,
  fetcher: fetch.Default(nil),
  }
  logger, _ := zap.NewProduction()
  engine.logger = logger.Sugar()
- engine.initActorProviders(timeout, flareSolverrURL)
- engine.initMovieProviders(timeout, flareSolverrURL)
+ engine.initActorProviders(timeout)
+ engine.initMovieProviders(timeout)
  return engine
 }
 
@@ -46,13 +45,13 @@ func Default() *Engine {
  DSN: "",
  DisableAutomaticPing: true,
  })
- engine := New(db, time.Minute, "")
+ engine := New(db, time.Minute)
  defer engine.AutoMigrate(true)
  return engine
 }
 
 // initActorProviders initializes actor providers.
-func (e *Engine) initActorProviders(timeout time.Duration, flareSolverrURL string) {
+func (e *Engine) initActorProviders(timeout time.Duration) {
  { // init
  e.actorProviders = make(map[string]mt.ActorProvider)
  e.actorHostProviders = make(map[string][]mt.ActorProvider)
@@ -62,9 +61,6 @@ func (e *Engine) initActorProviders(timeout time.Duration, flareSolverrURL strin
  if s, ok := provider.(mt.RequestTimeoutSetter); ok {
  s.SetRequestTimeout(timeout)
  }
- if s, ok := provider.(mt.FlareSolverrSetter); ok {
- s.SetFlareSolverr(flareSolverrURL)
- }
  // Add actor provider by name.
  e.actorProviders[strings.ToUpper(name)] = provider
  // Add actor provider by host.
@@ -74,7 +70,7 @@ func (e *Engine) initActorProviders(timeout time.Duration, flareSolverrURL strin
 }
 
 // initMovieProviders initializes movie providers.
-func (e *Engine) initMovieProviders(timeout time.Duration, flareSolverrURL string) {
+func (e *Engine) initMovieProviders(timeout time.Duration) {
  { // init
  e.movieProviders = make(map[string]mt.MovieProvider)
  e.movieHostProviders = make(map[string][]mt.MovieProvider)
@@ -84,9 +80,6 @@ func (e *Engine) initMovieProviders(timeout time.Duration, flareSolverrURL strin
  if s, ok := provider.(mt.RequestTimeoutSetter); ok {
  s.SetRequestTimeout(timeout)
  }
- if s, ok := provider.(mt.FlareSolverrSetter); ok {
- s.SetFlareSolverr(flareSolverrURL)
- }
  // Add movie provider by name.
  e.movieProviders[strings.ToUpper(name)] = provider
  // Add movie provider by host.
@@ -104,23 +97,6 @@ func (e *Engine) GetActorProviders() map[string]mt.ActorProvider {
  return e.actorProviders
 }
 
-func (e *Engine) GetActorProvidersByLanguage(lang string) (map[string]mt.ActorProvider, error) {
- tag, err := language.Parse(lang)
- if err != nil {
- return nil, err
- }
-
- providers := make(map[string]mt.ActorProvider)
- matcher := language.NewMatcher([]language.Tag{tag})
-
- for _, provider := range e.actorProviders {
- if _, _, c := matcher.Match(provider.Language()); c >= language.Low {
- providers[strings.ToUpper(provider.Name())] = provider
- }
- }
- return providers, nil
-}
-
 func (e *Engine) GetActorProviderByURL(rawURL string) (mt.ActorProvider, error) {
  u, err := url.Parse(rawURL)
  if err != nil {
@@ -159,23 +135,6 @@ func (e *Engine) GetMovieProviders() map[string]mt.MovieProvider {
  return e.movieProviders
 }
 
-func (e *Engine) GetMovieProvidersByLanguage(lang string) (map[string]mt.MovieProvider, error) {
- tag, err := language.Parse(lang)
- if err != nil {
- return nil, err
- }
-
- providers := make(map[string]mt.MovieProvider)
- matcher := language.NewMatcher([]language.Tag{tag})
-
- for _, provider := range e.movieProviders {
- if _, _, c := matcher.Match(provider.Language()); c >= language.Low {
- providers[strings.ToUpper(provider.Name())] = provider
- }
- }
- return providers, nil
-}
-
 func (e *Engine) GetMovieProviderByURL(rawURL string) (mt.MovieProvider, error) {
  u, err := url.Parse(rawURL)
  if err != nil {

engine/movie.go

@@ -85,7 +85,7 @@ func (e *Engine) SearchMovie(keyword, name string, fallback bool) ([]*model.Movi
  return e.searchMovie(keyword, provider, fallback)
 }
 
-func (e *Engine) searchMovieAll(keyword string, providers map[string]mt.MovieProvider) (results []*model.MovieSearchResult, err error) {
+func (e *Engine) searchMovieAll(keyword string) (results []*model.MovieSearchResult, err error) {
  type response struct {
  Results []*model.MovieSearchResult
  Error error
@@ -96,7 +96,7 @@ func (e *Engine) searchMovieAll(keyword string, providers map[string]mt.MoviePro
  respCh := make(chan response)
 
  var wg sync.WaitGroup
- for _, provider := range providers {
+ for _, provider := range e.movieProviders {
  wg.Add(1)
  // Goroutine started time.
  startTime := time.Now()
@@ -139,19 +139,11 @@ func (e *Engine) searchMovieAll(keyword string, providers map[string]mt.MoviePro
 }
 
 // SearchMovieAll searches the keyword from all providers.
-func (e *Engine) SearchMovieAll(keyword, lang string, fallback bool) (results []*model.MovieSearchResult, err error) {
+func (e *Engine) SearchMovieAll(keyword string, fallback bool) (results []*model.MovieSearchResult, err error) {
  if keyword = number.Trim(keyword); keyword == "" {
  return nil, mt.ErrInvalidKeyword
  }
 
- availableProviders := e.movieProviders
- if lang != "" {
- if availableProviders, err = e.GetMovieProvidersByLanguage(lang); err != nil {
- return
- }
- e.logger.Infof("Movie Keyword: %s, Language: %s, Providers: %v", keyword, lang, availableProviders)
- }
-
  defer func() {
  if err != nil {
  return
@@ -188,16 +180,12 @@ func (e *Engine) SearchMovieAll(keyword, lang string, fallback bool) (results []
  // overwrite error.
  err = nil
  // append results.
- for _, result := range innerResults {
- if _, ok := availableProviders[strings.ToUpper(result.Provider)]; ok {
- results = append(results, result)
- }
- }
+ results = append(results, innerResults...)
  }
  }()
  }
 
- results, err = e.searchMovieAll(keyword, availableProviders)
+ results, err = e.searchMovieAll(keyword)
  return
 }
 

go.mod

@@ -3,7 +3,6 @@ module github.com/metatube-community/metatube-sdk-go
 go 1.21
 
 require (
- github.com/SkYNewZ/go-flaresolverr v0.0.0-20230503120808-3826fdd382dd
  github.com/adrg/strutil v0.3.1
  github.com/antchfx/htmlquery v1.3.0
  github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
@@ -12,7 +11,6 @@ require (
  github.com/gin-gonic/gin v1.9.1
  github.com/glebarez/sqlite v1.10.0
  github.com/gocolly/colly/v2 v2.1.1-0.20230620150846-a6e3d81fe6b7
- github.com/google/uuid v1.5.0
  github.com/grafov/m3u8 v0.12.0
  github.com/hashicorp/go-retryablehttp v0.7.5
  github.com/iancoleman/orderedmap v0.3.0
@@ -56,6 +54,7 @@ require (
  github.com/goccy/go-json v0.10.2 // indirect
  github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
  github.com/golang/protobuf v1.5.3 // indirect
+ github.com/google/uuid v1.5.0 // indirect
  github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
  github.com/jackc/pgpassfile v1.0.0 // indirect
  github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect