190929 增加华文慕课解析功能

config/config.go

@@ -8,7 +8,10 @@ var (
  ContentOptions string
  // 显示版本号
  Version bool
+ // cookie 内容或者cookie文件地址
+ Cookie string
 )
+
 const VideoPPT="https://www.icourses.cn/web/sword/portal/shareChapter?cid="
 const Assignments="http://www.icourses.cn/web/sword/portal/assignments?cid="
 const TestPaper="http://www.icourses.cn/web/sword/portal/testPaper?cid="

download/download.go

@@ -16,13 +16,71 @@ import (
 
 
 //批量下载file数组里的文件
-func DownloadFiles(files []utils.File){
+func DownloadFiles(files []utils.File,courseName string){
  for _,file:=range(files){
  //下载到指定文件夹
- fmt.Println("\n"+filepath.Join(config.OutputPath,file.FilePATH))
- DownloadFile(file.FileURL,filepath.Join(config.OutputPath,file.FilePATH))
+ fmt.Println("\n"+filepath.Join(config.OutputPath,courseName,file.FilePATH))
+ DownloadFile(file.FileURL,filepath.Join(config.OutputPath,courseName,file.FilePATH))
  }
 }
+
+//使用cookie下载file文件
+func DownloadCookieFiles(files []utils.File,courseName string,cookie string){
+ for _,file:=range(files){
+ //下载到指定文件夹
+ fmt.Println("\n"+filepath.Join(config.OutputPath,courseName,file.FilePATH))
+ DownloadCookieFile(file.FileURL,filepath.Join(config.OutputPath,courseName,file.FilePATH), cookie)
+ }
+}
+
+//使用cookie下载单个文件
+func DownloadCookieFile(url string,filePath string,cookie string){
+ //fmt.Println(url)
+ //获取需要下载的文件大小
+ dataSize:=getFileSize(url)
+ //获取需要写入的信息
+ client := &http.Client{}
+ reqest, err := http.NewRequest("GET", url, nil)
+ if err != nil {
+ log.Fatal(err)
+ }
+ reqest.Header.Set("Cookie",cookie)
+ res, _ := client.Do(reqest)
+ defer res.Body.Close()
+ if res.StatusCode != 200 {
+ log.Fatalf("status code error: %d %s", res.StatusCode, res.Status)
+ }
+
+ //检查文件是否存在
+ if utils.FileExists(filePath){
+ fmt.Printf("file already exists, skipping\n")
+ return
+ }
+ //检查目录是否存在
+ if _, err := os.Stat(filepath.Dir(filePath)); os.IsNotExist(err) {
+ //建立目录
+ _ = os.MkdirAll(filepath.Dir(filePath), os.ModePerm)
+ }
+ //建立进程条,设置参数显示下载速度和下载进度
+ bar := progressbar.NewOptions(
+ int(dataSize),
+ progressbar.OptionSetBytes(int(dataSize)),
+ progressbar.OptionShowCount(),
+ progressbar.OptionShowIts(),
+ )
+
+ // 创建文件
+ dest, err := os.Create(filePath)
+ if err != nil {
+ fmt.Printf("Can't create %s: %v\n", filePath, err)
+ return
+ }
+ defer dest.Close()
+ // 从reader读入文件
+ out := io.MultiWriter(dest, bar)
+ _, _ = io.Copy(out, res.Body)
+}
+
 //根据网络URL获得文件的大小
 func getFileSize(url string) int64 {
  res, err := http.Head(url)
@@ -77,3 +135,4 @@ func DownloadFile(url string,filePath string){
  out := io.MultiWriter(dest, bar)
  _, _ = io.Copy(out, res.Body)
 }
+

main.go

@@ -16,40 +16,30 @@ import (
 func init() {
  flag.BoolVar(&config.Version, "v", false, "Show version")
  //all为全部下载，most为视频课件以及试卷，也为下载默认选项，videoPPT仅下载视频和课件，exams为仅下载试卷，resources仅下载其它资源
- flag.StringVar(&config.ContentOptions, "c", "most", "Specify the download content {all,most,videoPPT,assignments,testPaper,shareResource}")
+ flag.StringVar(&config.ContentOptions, "co", "all", "Only for icourse : Specify the download content {all,most,videoPPT,assignments,testPaper,shareResource}\nOnly for chinesemooc : Specify the download content {all, video , PPT}")
+ //华文慕课的下载选项，只有三个:全部下载，只下载视频 以及 只下载课件
+
+ //设置下载路径
  flag.StringVar(&config.OutputPath, "o", "", "Specify the output path")
- //flag.StringVar(&config.StartUrl, "F", "", "course URL")
+ //设置cookie
+ flag.StringVar(&config.Cookie, "c", "", "Cookie or the path of Cookie file")
 }
-func download(url string,options string) bool{
- id:=utils.MatchAll(url,`course_([0-9]*)`)
- if id != nil{
- //得到课程的id地址
- idNum:=id[0][1]
- //fmt.Println(idNum)
- switch options{
- case "all":
- parser.DownloadAll(idNum)
- case "most":
- parser.DownloadMost(idNum)
- case "videoPPT":
- parser.DownloadVideoPPT(idNum)
- case "assignments":
- parser.DownloadAssignments(idNum)
- case "testPaper":
- parser.DownloadTestPaper(idNum)
- case "shareResource":
- parser.DownloadShareResource(idNum)
- }
- } else{
- //网址不符合格式
- fmt.Printf("this website %s is not supported now",url)
- return true
+
+func download(url string) bool {
+ domain := utils.Domain(url)
+ switch domain {
+ case "icourses":
+ parser.DownloadIcourse(url,config.ContentOptions)
+ case "chinesemooc":
+ //fmt.Println("SUCCESS")
+ parser.DownloadChinesemooc(url,config.ContentOptions,config.Cookie)
  }
- return true
+ return true
 }
 
 func main() {
  //此处参考了annie的代码
+ //fmt.Println(parser.GetStartURLs("http://www.chinesemooc.org/mooc/4880","pku_auth=161evS%2BQJtmq%2FGJRyU%2BFhfaNLyG88SrUPqUX5a0eOUW49JVtBaPxY7lt1vp2MvvcC9UaH8qYx3%2B0cSja0MeVNCmDSWRQ; pku_loginuser=univeroner%40gmail.com; pku_reward_log=daylogin%2C1173273; Hm_lvt_ff4f6e9862a4e0e16fd1f5a7f6f8953b=1569321857,1569494843,1569494850,1569759380; PHPSESSID=p72d5gqftbmp65mmr2n9ghrah5; pku__refer=%252Fmooc%252F4880; Hm_lpvt_ff4f6e9862a4e0e16fd1f5a7f6f8953b=1569761588"))
 
  flag.Parse()
  args := flag.Args()
@@ -63,10 +53,14 @@ func main() {
  flag.PrintDefaults()
  return
  }
+ if config.Cookie != ""{
+ utils.ReadCookieFromFile(config.Cookie)
+ //fmt.Println(config.Cookie)
+ }
  var isErr bool
  //可以下载多个url
  for _, videoURL := range args {
- if err := download(strings.TrimSpace(videoURL),config.ContentOptions); err {
+ if err := download(strings.TrimSpace(videoURL)); err {
  isErr = true
  }
  }