<file_sep>package novelcrawler

// ErrorCode code of error type
type ErrorCode int

const (
	// Success success
	Success ErrorCode = iota
	// UnknownError error type
	UnknownError
	// RemoteNotExist remote url not exists
	RemoteNotExist
	// DBError the errors about database operation
	DBError
	// NovelName failed to get novel name
	NovelName
	// InternalError internal error
	InternalError
)

// ErrorString get the description of error code
func (code *ErrorCode) ErrorString() string {
	errors := [...]string{
		"Success",
		"Unknown error",
		"Remote novel not exists",
		"Database operation error",
		"Failed to get novel name",
		"Internal error",
	}
	if *code < Success || *code > InternalError {
		return "Wrong error code"
	}
	return errors[*code]
}
<file_sep>// +build !windows

package config

import (
	"os"
)

func getabspath() string {
	return getdbpath() + getdbname()
}

func getdbname() string {
	return "novel.db"
}

func getdbpath() string {
	return os.Getenv("HOME") + "/.novelCrawler/"
}

func getTempDir() string {
	return "/tmp/colly-cache"
}<file_sep>package novelcrawler

import (
	"fmt"
	"log"
	"strings"
)

// Chstonum returns the int number of chinese number,
// if unexpected number presents, or larger than 99999999,
// an error will be returned
func Chstonum(strnum string) (result int, err error) {
	var method func(int, *int)
	var containsBits bool
	if strings.Contains(strnum, "百") ||
		strings.Contains(strnum, "千") ||
		strings.Contains(strnum, "万") ||
		strings.Contains(strnum, "十") {
		method = withbits()
		containsBits = true
	} else {
		method = withoutbits()
		containsBits = false
	}
	pos := 0
	for _, ch := range strnum {
		pos, err = positionOf(ch)
		if err != nil {
			return pos, err
		}
		method(pos, &result)
		if result == -1 {
			log.Fatalf("Error: unexpcted number: %v(%v)", strnum, ch)
			return -1, fmt.Errorf("Error: unexpcted number: %v(%v)", strnum, ch)
		}
	}
	if containsBits && pos != 0 {
		result += pos
	}
	return
}

func withbits() func(int, *int) {
	tmp := -1
	return func(pos int, result *int) {
		if pos == 0 {
			tmp = -1
		} else if pos > 0 && pos < 10 {
			tmp = pos
		} else if pos >= 10 && pos < 10000 {
			if tmp < 0 && *result <= 0 {
				tmp = 1
			}
			*result += (tmp * pos)
			tmp = -1
		} else {
			if *result == 0 && tmp <= 0 {
				*result = -1
			} else if tmp > 0 {
				*result += tmp
			}
			*result *= pos
			tmp = -1
		}
	}
}

func withoutbits() func(int, *int) {
	return func(pos int, result *int) {
		if pos == 0 {
			*result *= 10
		} else if pos > 0 && pos < 10 {
			*result *= 10
			*result += pos
		} else {
			*result = -1
		}
	}
}

// positionOf return the position of a rune,
// if not in the slice, return an error
func positionOf(ch rune) (int, error) {
	if ch == '两' {
		ch = '二'
	}
	chsnums := []rune{'零', '一', '二', '三', '四', '五', '六', '七', '八', '九'}
	for pos, value := range chsnums {
		if ch == value {
			return pos, nil
		}
	}
	return bitToNumber(ch)
}

func bitToNumber(ch rune) (result int, err error) {
	switch ch {
	case '十':
		result = 10
	case '百':
		result = 100
	case '千':
		result = 1000
	case '万':
		result = 10000
	default:
		return -1, fmt.Errorf("Unexpcted chinese number: %v", ch)
	}
	return
}

// NumberToChs converts int number to chinese number
func NumberToChs(number int) string {
	chs := []string{"", "十", "百", "千", "万"}
	chsnums := []string{"零", "一", "二", "三", "四", "五", "六", "七", "八", "九"}
	var result string
	current := -1
	last := -1
	for index := 0; number > 0; index = (index + 1) % len(chs) {
		if current != -1 {
			last = current
			if index == 0 {
				index = 1
			}
		}
		current = number % 10
		number /= 10
		if current == 0 && (last == -1 || last == 0) {
			if index == 4 {
				result += chs[index]
			}
			continue
		} else if current == 0 && index != 4 {
			result = chsnums[current] + result
		} else if current == 0 && index == 4 {
			result = chs[index] + chsnums[current] + result
		} else {
			result = chsnums[current] + chs[index] + result
		}
	}
	return result
}
<file_sep>// +build windows 

package config

import (
	"os"
)

func getdbname() string {
	return "novel.db"
}

func getdbpath() string {
	return os.Getenv("APPDATA") + "\\novelCrawler\\"
}

func getabspath() string {
	return Getdbpath() + getdbname()
}

func getTempDir() string {
	return os.Getenv("TEMP") + "\\colly-cache"
}<file_sep>package crwaler

import (
	"log"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/imwyl/novelCrawler/config"
	"github.com/imwyl/novelCrawler/pkg/novelCrawler"

	"github.com/gocolly/colly"
	"github.com/gocolly/colly/proxy"
	"github.com/imwyl/novelCrawler/dao"
)

var chapterRegex, _ = regexp.Compile("(第[一二三四五六七八九零十百千万]+[章|节]\\s+.*)")
var novelNameRegex, _ = regexp.Compile("^(.*)最新章节$")
var novelURIRegex, _ = regexp.Compile(`\d+/\d+`)
var chapterURLRegex, _ = regexp.Compile("(\\d+)\\.html")
var firstChapterRegex, _ = regexp.Compile(`<a href="(\d+).html">第一[章|节]`)
var tempDir = config.GetTempDir()

func getChapterNumber(chapterURL string) int {
	if chapterURL == "" {
		return -1
	}
	chapterNumberArr := chapterURLRegex.FindStringSubmatch(chapterURL)
	if len(chapterNumberArr) <= 1 {
		log.Fatalln(chapterURL)
		return -1
	}
	result, err := strconv.Atoi(chapterNumberArr[1])
	if err != nil {
		return -1
	}
	return result
}

func getChapters(URL string) {
	var c = colly.NewCollector(
		colly.AllowedDomains("piaotian.net", "www.piaotian.net", "www.piaotian.com", "piaotian.com" ),
		colly.CacheDir(tempDir))
	c.Limit(&colly.LimitRule{
		Parallelism: 5,
	})
	enableProxy(c)
	d := c.Clone()
	enableProxy(d)
	d.Limit(&colly.LimitRule{
		Parallelism: 5,
	})
	db, err := dao.GetDB()
	if err != nil {
		log.Fatalln(err)
	}
	var chapter dao.Chapter
	novel := dao.Novel{ID: getNovelURI(URL)}
	db.First(&novel)
	db.Where("novel_id = ?", novel.ID).Order("id desc").First(&chapter)
	chapterMap := make(map[int]*dao.Chapter)
	// get chapters of novel
	c.OnHTML("ul", func(e *colly.HTMLElement) {
		e.ForEach("li", func(_ int, el *colly.HTMLElement) {
			if chapterRegex.MatchString(el.ChildText("a")) {
				chapterURL := el.ChildAttr("a", "href")
				if thisChapter := getChapterNumber(chapterURL); thisChapter-novel.First+1 > int(chapter.ID) {
					if chapterMap[thisChapter] == nil {
						chapterMap[thisChapter] = &dao.Chapter{
							ID:      uint(thisChapter),
							NovelID: novel.ID,
							Name:    chapterRegex.FindStringSubmatch(el.ChildText("a"))[1],
						}
						log.Printf("Add %d %v to map", thisChapter, chapterMap[thisChapter])
					}
					d.Visit(e.Request.AbsoluteURL(el.ChildAttr("a", "href")))
				}
			}
		})
	})

	// get content of novel
	d.OnHTML("div#main", func(e *colly.HTMLElement) {
		requestURL := chapterURLRegex.FindStringSubmatch(e.Request.URL.String())[1]
		chapterURL := getChapterNumber(requestURL)
		thisChapter := chapterMap[chapterURL]
		log.Println("div#main", thisChapter)
		thisChapter.Content = strings.Replace(e.Text, "\u00a0\u00a0\u00a0\u00a0", "<p>", -1)
		thisChapter.Save()
		//log.Println()
	})

	// div#main is add by javascript, so add it here
	d.OnResponse(func(res *colly.Response) {
		res.Body = []byte(strings.Replace(string(res.Body), "<br>", "<div id=\"main\">", 1))
		log.Println("Response code: ", string(res.Body))
	})

	c.Visit(URL)
	c.Wait()

}

// novelExits return the novel exits in database or not
func novelExists(URL string) bool {
	novelURI := getNovelURI(URL)
	log.Println(novelURI)
	return dao.NovelExists(novelURI)
}

// createNovel create novel record in database
func createNovel(URL string, boolChan chan novelcrawler.ErrorCode) {
	log.Println("URL", URL)
	db, getDbErr := dao.GetDB()
	if getDbErr != nil {
		log.Fatalln("Can not open databse:\n", getDbErr)
		panic(getDbErr)
	}
	defer db.Close()
	var c = colly.NewCollector(
		colly.AllowedDomains("www.piaotian.com", "piaotian.com", "piaotian.net", "www.piaotian.net"),
		colly.CacheDir(tempDir))
	enableProxy(c)
	var remoteNovelExist bool
	var firstChapter int
	c.OnResponse(func(res *colly.Response) {
		remoteNovelExist = res.StatusCode == 200
		if !remoteNovelExist {
			boolChan <- novelcrawler.RemoteNotExist
		} else {
			firstChapterSlice := firstChapterRegex.FindStringSubmatch(string(res.Body))
			if len(firstChapterSlice) <= 1 {
				boolChan <- novelcrawler.InternalError
			} else {
				firstChapter, _ = strconv.Atoi(firstChapterSlice[1])
			}
		}
	})
	c.OnHTML("h1", func(e *colly.HTMLElement) {
		h1title := e.Text
		// get novel's name and save to database
		if novelName := novelNameRegex.FindStringSubmatch(h1title); len(novelName) > 1 && remoteNovelExist {
			log.Println(novelName)
			novel := dao.Novel{
				ID:       getNovelURI(URL),
				Name:     novelName[1],
				First:    firstChapter,
				UpdateAt: time.Now(),
			}
			log.Println("create novel ", novel.Name)
			db.Create(&novel)
			boolChan <- novelcrawler.Success
		} else {
			boolChan <- novelcrawler.NovelName
			log.Println("novelExists ", remoteNovelExist)
		}
	})
	c.Visit(URL)
	c.Wait()
}

// Start the crawler
func Start(URL string) novelcrawler.ErrorCode {
	modifyURL(&URL)
	if !novelExists(URL) {
		errorChan := make(chan novelcrawler.ErrorCode)
		go createNovel(URL, errorChan)
		if createResult := <-errorChan; createResult != novelcrawler.Success {
			return createResult
		}
	}
	getChapters(URL)
	return novelcrawler.Success
}

func modifyURL(URL *string) {
	if !strings.Contains(*URL, "https://www.") {
		*URL = "https://www." + *URL
	}
	if strings.HasSuffix(*URL, "/") {
		*URL = string((*URL)[:len(*URL)-1])
	} else {
		*URL = strings.Replace(*URL, "/index.html", "", 1)
	}
}

func enableProxy(c *colly.Collector) {
	p, proxyErr := proxy.RoundRobinProxySwitcher("socks5://127.0.0.1:9050")
	if proxyErr != nil {
		panic(proxyErr)
	}
	c.SetProxyFunc(p);
}
func getNovelURI(URL string) string {
	return novelURIRegex.FindString(URL)
}
<file_sep>package config

import (
	"log"
	"os"
)

// Getdbname returns the name of db file
func Getdbname() string {
	return getdbname()
}

// Getdbpath returns the path of db file 
func Getdbpath() string {
	path := getdbpath()
	_, err := os.Stat(path)
	if err != nil {
		result := os.MkdirAll(path, 777)
		if result != nil {
			log.Fatalf("Create dir %s failed", path)
			return ""
		}
		log.Println("Create dir:", path)
	}
	return path
}

// Getabspath return absolute path of db file
func Getabspath() string {
	return getabspath()
}

// GetTempDir returns temp directory on diffrent OS
func GetTempDir() string {
	return getTempDir()
}<file_sep>package novelcrawler

import (
	nc "github.com/imwyl/novelCrawler/config"
	"testing"
)

func TestGetdbname(t *testing.T) {
	tests := []struct {
		name string
		want string
	}{
		{"test1", ".novel.db"},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := nc.Getdbname(); got != tt.want {
				t.Errorf("Getdbname() = %v, want %v", got, tt.want)
			}
		})
	}
}

func TestGetdbpath(t *testing.T) {
	tests := []struct {
		name string
		want string
	}{
		{"test2", "C:\\Users\\IMwyl\\AppData\\Roaming\\novelCrawler\\"},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := nc.Getdbpath(); got != tt.want {
				t.Errorf("Getdbpath() = %v, want %v", got, tt.want)
			}
		})
	}
}

func TestGetabspath(t *testing.T) {
	tests := []struct {
		name string
		want string
	}{
		{"test3", "/home/wyl/.novel.db"},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := nc.Getabspath(); got != tt.want {
				t.Errorf("Getabspath() = %v, want %v", got, tt.want)
			}
		})
	}
}

func TestGetTempDir(t *testing.T) {
	tests := []struct {
		name string
		want string
	}{
		{"test3", "C:\\Users\\IMwyl\\AppData\\Local\\Temp\\colly-cache"},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := nc.GetTempDir(); got != tt.want {
				t.Errorf("Getabspath() = %v, want %v", got, tt.want)
			}
		})
	}
}<file_sep>package novelcrawler

import (
	"testing"

	"github.com/imwyl/novelCrawler/pkg/novelCrawler"
)

func TestChstonum(t *testing.T) {
	type args struct {
		strnum string
	}
	tests := []struct {
		name       string
		args       args
		wantResult int
		wantErr    bool
	}{
		{"0", args{"零"},
			0, false},
		{"1", args{"一"},
			1, false},
		{"2", args{"十二"},
			12, false},
		{"3", args{"一百零三"},
			103, false},
		{"3", args{"一零三"},
			103, false},
		{"4", args{"一百四十五"},
			145, false},
		{"5-1", args{"六千零七十八"},
			6078, false},
		{"5-3", args{"六千零一"},
			6001, false},
		{"5-2", args{"六零七八"},
			6078, false},
		{"6", args{"九千一百零二"},
			9102, false},
		{"7", args{"三千四百五十六"},
			3456, false},
		{"8", args{"七万零八"},
			70008, false},
		{"9", args{"九万零一百零二"},
			90102, false},
		{"10", args{"三万零四百五十六"},
			30456, false},
		{"11", args{"七万八千零九"},
			78009, false},
		{"12", args{"十万两千三百零四"},
			102304, false},
		{"13", args{"五十万六千七百八十九"},
			506789, false},
		{"13-1", args{"五零六七八九"},
			506789, false},
		{"14", args{"一百二十三万四千五百六十七"},
			1234567, false},
		{"err", args{"第一章"},
			-1, true},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			gotResult, err := novelcrawler.Chstonum(tt.args.strnum)
			if (err != nil) != tt.wantErr {
				t.Errorf("Chstonum() error = %v, wantErr %v", err, tt.wantErr)
				return
			}
			if gotResult != tt.wantResult {
				t.Errorf("Chstonum() = %v, want %v", gotResult, tt.wantResult)
			}
		})
	}
}

func TestNumberToChs(t *testing.T) {
	type args struct {
		number int
	}
	tests := []struct {
		name string
		args args
		want string
	}{
		{"1", args{1200}, "一千二百"},
		{"2", args{10}, "一十"},
		{"3", args{14}, "一十四"},
		{"4", args{20}, "二十"},
		{"5", args{100}, "一百"},
		{"6", args{104}, "一百零四"},
		{"7", args{112}, "一百一十二"},
		{"8", args{130}, "一百三十"},
		{"9", args{1000}, "一千"},
		{"10", args{1002}, "一千零二"},
		{"11", args{1030}, "一千零三十"},
		{"12", args{4056}, "四千零五十六"},
		{"13", args{7890}, "七千八百九十"},
		{"14", args{1200}, "一千二百"},
		{"15", args{102345}, "一十万零二千三百四十五"},
		{"16", args{40000}, "四万"},
		{"17", args{500000}, "五十万"},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := novelcrawler.NumberToChs(tt.args.number); got != tt.want {
				t.Errorf("NumberToChs() = %v, want %v", got, tt.want)
			}
		})
	}
}
<file_sep>package dao

import (
	"time"

	"github.com/imwyl/novelCrawler/config"
	"github.com/jinzhu/gorm"
)

// Novel the novels
type Novel struct {
	ID			string		`grom:"primary_key; auto_increment:false"`
	Name		string		`gorm:"type:varchar(100);not null"`
	Chapters	[]Chapter	`gorm:"foreignkey:NovelID"`
	First		int			`gorm:"not null"`
	UpdateAt	time.Time	`gorm:"not null"`
}

// Chapter the chapters of volumn
type Chapter struct {
	ID      uint	`gorm:"primary_key;auto_increment:false"`
	Name    string
	NovelID string `gorm:"not null"`
	Content string `gorm:"type:Text(100000)"`
}

// GetDB returns a database connection
func GetDB() (db *gorm.DB, err error) {
	db, err = gorm.Open("sqlite3", config.Getabspath())
	return
}

// Save a chapter entity
func (chapter *Chapter) Save() (bool, error) {
	db, err := GetDB()
	if err != nil {
		return false, err
	}
	var count int
	if db.Model(chapter).Where("id = ?", chapter.ID).Count(&count); count == 0 {
		db.Create(chapter)
	} else {
		db.Save(chapter)
	}
	var novel Novel
	db.Model(chapter).Related(&novel)
	db.Model(&novel).Update("update_at", time.Now())
	return true, nil
}

// NovelExists check a record exists or not
func NovelExists(id string) bool {
	db, err := GetDB()
	if err != nil {
		panic(err)
	}
	defer db.Close()
	var count int
	db.Model(&Novel{}).Where("id = ?", id).Count(&count)
	return count > 0
}
<file_sep>package main

import (
	"strings"
	"flag"
	"log"

	"github.com/imwyl/novelCrawler/dao"

	"github.com/imwyl/novelCrawler/crwaler"
	_ "github.com/jinzhu/gorm/dialects/sqlite"
)

func main() {
	var URL string
	flag.StringVar(&URL, "URL", "", "飘天小说地址")
	flag.Parse()
	if URL == "" {
		log.Fatalln("No URL")
		return
	} else if !strings.Contains(URL, "piaotian") {
		log.Fatalln("Incorrent URL")
	}
	db, err := dao.GetDB()
	if err != nil {
		log.Fatalln(err)
		return
	}
	defer db.Close()
	db.AutoMigrate(&dao.Chapter{}, &dao.Novel{})
	crwaler.Start(URL)
}
<file_sep>package crwaler

import "testing"

func Test_chapterOrder(t *testing.T) {
	type args struct {
		chapterURL string
	}
	tests := []struct {
		name string
		args args
		want int
	}{
		{"1", args{"1234221.html"}, 1234221},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := getChapterNumber(tt.args.chapterURL); got != tt.want {
				t.Errorf("chapterOrder() = %v, want %v", got, tt.want)
			}
		})
	}
}
