diff --git a/conf/app.conf.example b/conf/app.conf.example index 9b60c560..f11d0fe0 100644 --- a/conf/app.conf.example +++ b/conf/app.conf.example @@ -74,7 +74,12 @@ mail_expired=30 secure=LOGIN ###############配置PDF生成工具地址################### -wkhtmltopdf=D:/Program Files/wkhtmltopdf/bin/wkhtmltopdf.exe +#同一个项目同时运行导出程序的并行数量,取值1-4之间,取值越大导出速度越快,越占用资源 +export_process_num=1 +#并发导出的项目限制,指同一时间限制的导出项目数量,如果为0则不限制。设置的越大,越占用资源 +export_limit_num=1 +#指同时等待导出的任务数量 +export_queue_limit_num=100 ###############配置CDN加速################## cdn= diff --git a/conf/enumerate.go b/conf/enumerate.go index 1409b6f2..44708bea 100644 --- a/conf/enumerate.go +++ b/conf/enumerate.go @@ -135,6 +135,33 @@ func GetUploadFileSize() int64 { } return 0 } +//同一项目导出线程的并发数 +func GetExportProcessNum() int { + exportProcessNum := beego.AppConfig.DefaultInt("export_process_num",1) + + if exportProcessNum <= 0 || exportProcessNum > 4 { + exportProcessNum = 1 + } + return exportProcessNum; +} +//导出项目队列的并发数量 +func GetExportLimitNum() int { + exportLimitNum := beego.AppConfig.DefaultInt("export_limit_num",1) + + if exportLimitNum < 0 { + exportLimitNum = 1 + } + return exportLimitNum; +} +//等待导出队列的长度 +func GetExportQueueLimitNum() int { + exportQueueLimitNum := beego.AppConfig.DefaultInt("export_queue_limit_num",10) + + if exportQueueLimitNum <= 0 { + exportQueueLimitNum = 100 + } + return exportQueueLimitNum +} //判断是否是允许商城的文件类型. func IsAllowUploadFileExt(ext string) bool { diff --git a/controllers/BaseController.go b/controllers/BaseController.go index 148ea340..a19072db 100644 --- a/controllers/BaseController.go +++ b/controllers/BaseController.go @@ -154,7 +154,7 @@ func (c *BaseController) ShowErrorPage(errCode int, errMsg string) { var buf bytes.Buffer - if err := beego.ExecuteViewPathTemplate(&buf, "document/export.tpl", beego.BConfig.WebConfig.ViewsPath, map[string]interface{}{"ErrorMessage": errMsg, "errCode": errCode, "BaseUrl": conf.BaseUrl}); err != nil { + if err := beego.ExecuteViewPathTemplate(&buf, "errors/error.tpl", beego.BConfig.WebConfig.ViewsPath, map[string]interface{}{"ErrorMessage": errMsg, "ErrorCode": errCode, "BaseUrl": conf.BaseUrl}); err != nil { c.Abort("500") } diff --git a/controllers/DocumentController.go b/controllers/DocumentController.go index b158929c..2a738a0f 100644 --- a/controllers/DocumentController.go +++ b/controllers/DocumentController.go @@ -28,6 +28,7 @@ import ( "gopkg.in/russross/blackfriday.v2" "github.com/lifei6671/mindoc/utils/cryptil" "fmt" + "github.com/lifei6671/mindoc/utils/filetil" ) // DocumentController struct @@ -857,29 +858,32 @@ func (c *DocumentController) Export() { return } - eBookResult, err := bookResult.Converter(c.CruSession.SessionID()) + outputPath := filepath.Join(conf.WorkingDirectory, "uploads", "books", strconv.Itoa(bookResult.BookId)) - if err != nil { - beego.Error("转换文档失败:" + bookResult.BookName + " -> " + err.Error()) - c.Abort("500") - } + pdfpath := filepath.Join(outputPath, "book.pdf") + epubpath := filepath.Join(outputPath, "book.epub") + mobipath := filepath.Join(outputPath, "book.mobi") + docxpath := filepath.Join(outputPath, "book.docx") - if output == "pdf" { - c.Ctx.Output.Download(eBookResult.PDFPath, bookResult.BookName+".pdf") + if output == "pdf" && filetil.FileExists(pdfpath){ + c.Ctx.Output.Download(pdfpath, bookResult.BookName+".pdf") + c.Abort("200") + } else if output == "epub" && filetil.FileExists(epubpath){ + c.Ctx.Output.Download(epubpath, bookResult.BookName+".epub") c.Abort("200") - } else if output == "epub" { - c.Ctx.Output.Download(eBookResult.EpubPath, bookResult.BookName+".epub") + } else if output == "mobi" && filetil.FileExists(mobipath) { + c.Ctx.Output.Download(mobipath, bookResult.BookName+".mobi") c.Abort("200") - } else if output == "mobi" { - c.Ctx.Output.Download(eBookResult.MobiPath, bookResult.BookName+".mobi") + } else if output == "docx" && filetil.FileExists(docxpath){ + c.Ctx.Output.Download(docxpath, bookResult.BookName+".docx") c.Abort("200") - } else if output == "docx" { - c.Ctx.Output.Download(eBookResult.WordPath, bookResult.BookName+".docx") - c.Abort("200") + }else if output == "pdf" || output == "epub" || output == "docx" || output == "mobi"{ + models.BackgroupConvert(c.CruSession.SessionID(),bookResult) + c.ShowErrorPage(200,"文档正在后台转换,请稍后再下载") }else{ c.ShowErrorPage(200,"不支持的文件格式") } diff --git a/converter/converter.go b/converter/converter.go index bf16512f..93aae977 100644 --- a/converter/converter.go +++ b/converter/converter.go @@ -18,6 +18,7 @@ import ( "github.com/lifei6671/mindoc/utils/ziptil" "github.com/lifei6671/mindoc/utils/cryptil" "sync" + "html" ) type Converter struct { @@ -26,6 +27,9 @@ type Converter struct { Config Config Debug bool GeneratedCover string + ProcessNum int //并发的任务数量 + process chan func() + limitChan chan bool } //目录结构 @@ -93,6 +97,9 @@ func NewConverter(configFile string, debug ...bool) (converter *Converter, err e Config: cfg, BasePath: basepath, Debug: db, + ProcessNum: 1, + process: make(chan func(),4), + limitChan: make(chan bool,1), } } } @@ -100,89 +107,113 @@ func NewConverter(configFile string, debug ...bool) (converter *Converter, err e } //执行文档转换 -func (this *Converter) Convert() (err error) { - if !this.Debug { //调试模式下不删除生成的文件 - defer this.converterDefer() //最后移除创建的多余而文件 +func (convert *Converter) Convert() (err error) { + if !convert.Debug { //调试模式下不删除生成的文件 + defer convert.converterDefer() //最后移除创建的多余而文件 + } + if convert.process == nil{ + convert.process = make(chan func(),4) + } + if convert.limitChan == nil { + if convert.ProcessNum <= 0 { + convert.ProcessNum = 1 + } + convert.limitChan = make(chan bool,convert.ProcessNum) + for i := 0; i < convert.ProcessNum;i++{ + convert.limitChan <- true + } } - if err = this.generateMimeType(); err != nil { + if err = convert.generateMimeType(); err != nil { return } - if err = this.generateMetaInfo(); err != nil { + if err = convert.generateMetaInfo(); err != nil { return } - if err = this.generateTocNcx(); err != nil { //生成目录 + if err = convert.generateTocNcx(); err != nil { //生成目录 return } - if err = this.generateSummary(); err != nil { //生成文档内目录 + if err = convert.generateSummary(); err != nil { //生成文档内目录 return } - if err = this.generateTitlePage(); err != nil { //生成封面 + if err = convert.generateTitlePage(); err != nil { //生成封面 return } - if err = this.generateContentOpf(); err != nil { //这个必须是generate*系列方法的最后一个调用 + if err = convert.generateContentOpf(); err != nil { //这个必须是generate*系列方法的最后一个调用 return } //将当前文件夹下的所有文件压缩成zip包,然后直接改名成content.epub - f := filepath.Join(this.OutputPath, "content.epub") + f := filepath.Join(convert.OutputPath, "content.epub") os.Remove(f) //如果原文件存在了,则删除; - if err = ziptil.Zip(this.BasePath,f); err == nil { + if err = ziptil.Zip(convert.BasePath,f); err == nil { //创建导出文件夹 - os.Mkdir(this.BasePath+"/"+output, os.ModePerm) - if len(this.Config.Format) > 0 { + os.Mkdir(convert.BasePath+"/"+output, os.ModePerm) + if len(convert.Config.Format) > 0 { var errs []string - group := sync.WaitGroup{} - for _, v := range this.Config.Format { - fmt.Println("convert to " + v) - switch strings.ToLower(v) { - case "epub": - group.Add(1) - go func(group *sync.WaitGroup) { - if err = this.convertToEpub(); err != nil { - errs = append(errs, err.Error()) - fmt.Println("转换EPUB文档失败:" + err.Error()) - } - group.Done() - }(&group) - - case "mobi": - group.Add(1) - go func(group *sync.WaitGroup) { - if err = this.convertToMobi(); err != nil { - errs = append(errs, err.Error()) - fmt.Println("转换MOBI文档失败:" + err.Error()) - } - group.Done() - }(&group) - case "pdf": - group.Add(1) - go func(group *sync.WaitGroup) { - if err = this.convertToPdf(); err != nil { - fmt.Println("转换PDF文档失败:" + err.Error()) - errs = append(errs, err.Error()) - } - group.Done() - }(&group) - case "docx": - group.Add(1) - go func(group *sync.WaitGroup) { - if err = this.convertToDocx(); err != nil { - fmt.Println("转换WORD文档失败:" + err.Error()) - errs = append(errs, err.Error()) + go func(convert *Converter) { + for _, v := range convert.Config.Format { + fmt.Println("convert to " + v) + switch strings.ToLower(v) { + case "epub": + convert.process <- func() { + if err = convert.convertToEpub(); err != nil { + errs = append(errs, err.Error()) + fmt.Println("转换EPUB文档失败:" + err.Error()) + } } - group.Done() - }(&group) + case "mobi": + convert.process <- func() { + if err = convert.convertToMobi(); err != nil { + errs = append(errs, err.Error()) + fmt.Println("转换MOBI文档失败:" + err.Error()) + } + } + case "pdf": + convert.process <- func() { + if err = convert.convertToPdf(); err != nil { + fmt.Println("转换PDF文档失败:" + err.Error()) + errs = append(errs, err.Error()) + } + } + case "docx": + convert.process <- func() { + if err = convert.convertToDocx(); err != nil { + fmt.Println("转换WORD文档失败:" + err.Error()) + errs = append(errs, err.Error()) + } + } + } } + close(convert.process) + }(convert) + + group := sync.WaitGroup{} + for { + action, isClosed := <-convert.process + fmt.Println(action,isClosed) + if action == nil && !isClosed { + break; + } + group.Add(1) + <- convert.limitChan + fmt.Println("正在处理") + go func(group *sync.WaitGroup) { + action() + group.Done() + convert.limitChan <- true + }(&group) } + group.Wait() + if len(errs) > 0 { err = errors.New(strings.Join(errs, "\n")) } } else { - err = this.convertToPdf() + err = convert.convertToPdf() if err != nil { fmt.Println(err) } @@ -272,7 +303,7 @@ func (this *Converter) generateTocNcx() (err error) { ` codes, _ := this.tocToXml(0, 1) - ncx = fmt.Sprintf(ncx, this.Config.Language, this.Config.Title, strings.Join(codes, "")) + ncx = fmt.Sprintf(ncx, this.Config.Language, html.EscapeString(this.Config.Title), strings.Join(codes, "")) return ioutil.WriteFile(filepath.Join(this.BasePath, "toc.ncx"), []byte(ncx), os.ModePerm) } @@ -327,11 +358,11 @@ func (this *Converter) tocToSummary(pid int) (summarys []string) { summarys = append(summarys, "