<repo_name>astroshot/mini-spider-golang<file_sep>/src/main/mini_spider.go
/* mini_spider.go - program entry point */
/*
modification history
--------------------
2017/07/20, by <NAME>, create
*/
/*
DESCRIPTION
mini spider
*/

package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"
	"syscall"
	"time"
)


import (
	"code.google.com/p/log4go"
	"www.baidu.com/golang-lib/log"
)

import (
	"mini_spider_config"
	"mini_spider"
)

var (
	confPath    *string = flag.String("c", "../conf/spider.conf", "mini_spider configure path")
	help        *bool   = flag.Bool("h", false, "show help")
	logPath     *string = flag.String("l", "../log", "dir path of log")
	showVer     *bool   = flag.Bool("v", false, "show version")
	stdOut      *bool   = flag.Bool("s", false, "to show log in stdout")
	debugLog    *bool   = flag.Bool("d", false, "to show debug log (otherwise >= info)")
)

func Exit(code int) {
	log.Logger.Close()
	/* to overcome bug in log, sleep for a while    */
	time.Sleep(1 * time.Second)
	os.Exit(code)
}

/* the main function */
func main() {
	var logSwitch string

	flag.Parse()
	if *help {
		flag.PrintDefaults()
		return
	}
	if *showVer {
		fmt.Printf("version is: 1.0.0\n")
		return
	}

	// debug switch
	if *debugLog {
		logSwitch = "DEBUG"
	} else {
		logSwitch = "INFO"
	}
	fmt.Printf("mini_spider starts...\n")

	/* initialize log   */
	/* set log buffer size  */
	log4go.SetLogBufferLength(10000)
	/* if blocking, log will be dropped */
	log4go.SetLogWithBlocking(false)
	/* we want to get state of log4go   */
	log4go.SetWithModuleState(true)

	err := log.Init("mini_spider", logSwitch, *logPath, *stdOut, "midnight", 5)
	if err != nil {
		fmt.Printf("main(): err in log.Init():%s\n", err.Error())
		os.Exit(-1)
	}

	// load config
	config, err := mini_spider_config.LoadConfig(*confPath)
	if err != nil {
		log.Logger.Error("main():err in ConfigLoad():%s", err.Error())
		Exit(-1)
	}

	// load seeds
	seeds, err := mini_spider.LoadSeedFile(config.Basic.UrlListFile)
	if err != nil {
		log.Logger.Error("main():err in loadSeedFile(%s):%s", config.Basic.UrlListFile, err.Error())
		Exit(1)
	}

	// create mini-spider
	miniSpider, err:= mini_spider.NewMiniSpider(&config, seeds)
	if err != nil {
		log.Logger.Error("main():err in NewMiniSpider():%s", err.Error())
		Exit(1)
	}

	// run mini-spider
	miniSpider.Run()

	// waiting for all tasks to finish. 
	go func() {
		for {
			if miniSpider.GetUnfinished() == 0 {
				log.Logger.Info("All task finished, quit")
				Exit(0)
			}

			log.Logger.Debug("Waiting for %d tasks to finish\n", miniSpider.GetUnfinished())

			// sleep for a while
			time.Sleep(5 * time.Second)
		}
	} ()
	
	// Handle SIGINT and SIGTERM.
	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
	<-ch

	// ensure that all logs are export and normal exit
	Exit(0)

}
<file_sep>/src/web_package/webpage_parse.go
/* webpage_parse.go - parse urls from web page	*/
/*
modification history
--------------------
2017/07/18, by <NAME>, create
*/
/*
DESCRIPTION
*/
package web_package

import (
	"bytes"
	"fmt"
	"net/url"
)

import (
	"golang.org/x/net/html"
)

type HtmlLinks struct {
	links []string
}

// create new HtmlLinks
func NewHtmlLinks() *HtmlLinks {
	hl := new(HtmlLinks)
	hl.links = make([]string, 0)
	return hl
}

/*
get all href in given html node

Params:
	- n: html node
	- refUrl: reference url
*/
func (hl *HtmlLinks) getLinks(n *html.Node, refUrl *url.URL) {
	if n.Type == html.ElementNode && n.Data == "a" {
		for _, a := range n.Attr {
			if a.Key == "href" {
				linkUrl, err := refUrl.Parse(a.Val)
				if err == nil {
					hl.links = append(hl.links, linkUrl.String())
				}
				break
			}
		}
	}

	for c := n.FirstChild; c != nil; c = c.NextSibling {
		hl.getLinks(c, refUrl)
	}
}

/*
get url links in given html page

Params:
	- data: data for html page
	- urlStr: url string of this html page

Returns:
	- links: parsed links
	- error: any failure
*/
func ParseWebPage(data []byte, urlStr string) ([]string, error) {
	// parse html
	doc, err := html.Parse(bytes.NewReader(data))
	if err != nil {
		return nil, fmt.Errorf("html.Parse():%s", err.Error())
	}

	// parse url
	refUrl, err := url.ParseRequestURI(urlStr)
	if err != nil {
		return nil, fmt.Errorf("url.ParseRequestURI(%s):%s", urlStr, err.Error())
	}
	
	// get all links
	hl := NewHtmlLinks()
	hl.getLinks(doc, refUrl)

	return hl.links, nil
}
<file_sep>/src/mini_spider/url_table_test.go
/* url_table_test.go - test for url_table.go  */
/*
modification history
--------------------
2017/07/23, by <NAME>, create
*/
/*
DESCRIPTION
*/
package mini_spider

import (
	"testing"
)

func TestUrlTable_case_1(t *testing.T) {
	// create table
	ut := NewUrlTable()

	// check whether exist
	if ut.Exist("www.baidu.com") {
		t.Errorf("www.baidu.com should not exist")
	}

	// add to table
	ut.Add("www.baidu.com")

	// check whether exist
	if !ut.Exist("www.baidu.com") {
		t.Errorf("www.baidu.com should exist")
	}

	if ut.Exist("www.sina.com.cn") {
		t.Errorf("www.sina.com.cn should not exist")
	}
}
<file_sep>/src/mini_spider/webpage_parse_test.go
/* webpage_parse_test.go - test for webpage_parse.go  */
/*
modification history
--------------------
2017/07/22, by <NAME>, create
*/
/*
DESCRIPTION
*/
package mini_spider

import (
	"testing"
)

// test for parseWebPage()
func TestParseWebPage(t *testing.T) {
	s := []byte(`<p>Links:</p><ul><li><a href="test">Test</a><li><a href="/test1/test2">Test1_est2</a></ul>`)

	links, err := parseWebPage(s, "http://www.baidu.com/a/b.html")
	if err != nil {
		t.Errorf("err in parseWebPage():%s", err.Error())
		return
	}

	if len(links) != 2 {
		t.Errorf("len(links) should be 2, now it's %d", len(links))
		return
	}

	if links[0] != "http://www.baidu.com/a/test" || links[1] != "http://www.baidu.com/test1/test2" {
		t.Errorf("links:%s", links)
	}
}
<file_sep>/build.sh
#!/usr/bin/bash

### dir structure
# /bfe
#     /bfe-common
#         /go
#             /output
#         /golang-lib
#         /mini_spider
#             build.sh

### restore working dir
WORKROOT=$(pwd) 

cd ${WORKROOT}

# prepare PATH, GOROOT and GOPATH
export GOPATH=$(pwd)

# export golang-lib to GOPATH
cd ${WORKROOT}
export GOPATH=$(pwd)/../bfe-common/golang-lib:$GOPATH

# run go test for all subdirectory
cd ${WORKROOT}/src/mini_spider
go test -c -o ./testRun
if [ $? -ne 0 ];
then
    echo "go compile test failed"
    exit 1
fi

go test -run testRun
if [ $? -ne 0 ];
then
    echo "go run test failed"
    exit 1
fi
rm -rf ./testRun
echo "OK for go test"

### build 
cd ${WORKROOT}/src/main
go build -o mini_spider
if [ $? -ne 0 ];
then
    echo "fail to go build mini_spider.go"
    exit 1
fi
echo "OK for go build mini_spider.go"

### create directory for output
cd ../../
if [ -d "./output" ]
then
    rm -rf output
fi
mkdir output

# copy config
mkdir output/conf
cp conf/spider.conf output/conf

# copy data
cp -r data output/

# copy file to bin
mkdir output/bin
mv src/main/mini_spider output/bin

# create dir for log
mkdir output/log

# change mode of files in /bin
chmod +x output/bin/mini_spider


echo "OK for build mini_spider"
<file_sep>/src/mini_spider/queue.go
/* queue.go - FIFO */
/*
modification history
--------------------
2017/7/17, by linxiongmin, create
*/
/*
DESCRIPTION
*/
package mini_spider

import (
	"container/list"
	"errors"
	"sync"
)

/* max queue length */
const (
    MAX_QUEUE_LEN = 65535
)

/* queue */
type Queue struct {
	lock   sync.Mutex
	cond   *sync.Cond
	tasks  *list.List
	maxLen int         // max queue length
	unfinished int	   // number of unfinished tasks
}

/* Initialize the queue */
func (q *Queue) Init() {
	q.cond = sync.NewCond(&q.lock)
	q.tasks = list.New()
	q.maxLen = MAX_QUEUE_LEN
}

/* Add to the queue */
func (q *Queue) Add(task *CrawlTask) error {
	q.cond.L.Lock()
	defer q.cond.L.Unlock()

	var err error

	if q.tasks.Len() >= q.maxLen {
		err = errors.New("Queue is full")
	} else {
		q.tasks.PushBack(task)
		q.unfinished += 1
		q.cond.Signal()
		err = nil
	}
	
	return err
}

/* pop a task from the queue */
func (q *Queue) Pop() *CrawlTask {
	q.cond.L.Lock()
	defer q.cond.L.Unlock()

	for q.tasks.Len() == 0 {
		q.cond.Wait()
	}

	task := q.tasks.Front()
	q.tasks.Remove(task)

	return task.Value.(*CrawlTask)
}

/* Get length of the queue */
func (q *Queue) Len() int {
	q.lock.Lock()
	defer q.lock.Unlock()

	var len int
	len = q.tasks.Len()
	
	return len
}

/* Finish one task */
func (q *Queue) FinishOneTask() {
	q.lock.Lock()
	defer q.lock.Unlock()

	q.unfinished -= 1
}

/* get count of unfinished tasks */
func (q *Queue) GetUnfinished() int {
	q.lock.Lock()
	defer q.lock.Unlock()
	
	ret := q.unfinished
	
	return ret
}

/* set max queue length */
func (q *Queue) SetMaxLen(maxLen int) {
	q.lock.Lock()
	defer q.lock.Unlock()

	q.maxLen = maxLen
}
<file_sep>/src/mini_spider/queue_test.go
/* queue_test.go - test for queue.go  */
/*
modification history
--------------------
2017/07/23, by <NAME>, create
*/
/*
DESCRIPTION
*/
package mini_spider

import (
	"testing"
)

func TestQueue(t *testing.T) {
	var queue Queue
	queue.Init()

	queue.Add(&CrawlTask{"http://www.baidu.com", 2, nil})
	queue.Add(&CrawlTask{"http://www.sina.com", 1, nil})
	
	queue.SetMaxLen(2)

	err := queue.Add(&CrawlTask{"http://www.test.com", 1, nil})

	if err == nil {
		t.Error("queue is full, qeueu.Add should occur an error")
	}

	task := queue.Pop()
	if task == nil || task.Url != "http://www.baidu.com" {
        t.Errorf("www.baidu.com should be poped first")
    }

	qLen := queue.Len()
	if qLen != 1 {
		t.Error("queue.Len() should be 1, now it's %d", qLen)
	}
	
	n := queue.GetUnfinished()
	if n != 2 {
		t.Error("queue.GetUnfinished() should be 2, now it's %d", n)
	}
	
	queue.FinishOneTask()
	n = queue.GetUnfinished()
	if n != 1 {
		t.Error("queue.GetUnfinished() should be 1, now it's %d", n)
	}

	task = queue.Pop()
	if task == nil || task.Url != "http://www.sina.com" || task.Depth != 1 {
		t.Error("queue should pop http://www.sina.com with depth 1")
	}

	task = queue.Pop()
	if task != nil {
		t.Error("queue should pop nil")
	}

	queue.FinishOneTask()
	n = queue.GetUnfinished()
	if n != 0 {
		t.Error("all tasks in queue had finished, GetUnfinished() should return 0")
	}

}<file_sep>/src/mini_spider/mini_spider.go
/* mini_spider.go */
/*
modification history
--------------------
2017/07/20, by <NAME>, create
*/
/*
DESCRIPTION
- mini spider
*/

package mini_spider

import (
	"mini_spider_config"
)

type MiniSpider struct {
	config   *mini_spider_config.MiniSpiderConf
	urlTable *UrlTable
	queue    Queue
	crawlers []*Crawler
}

// crawl task
type CrawlTask struct {
	Url    string             // url to crawl
	Depth  int                // depth of the url
	Header map[string]string  // http header
}

// create new mini-spider
func NewMiniSpider(conf *mini_spider_config.MiniSpiderConf, seeds []string) (*MiniSpider, error) {
	ms := new(MiniSpider)
	ms.config = conf

	// create url table
	ms.urlTable = NewUrlTable()

	// initialize queue
	ms.queue.Init()

	// add seeds to queue
	for _, seed := range seeds {
		task := &CrawlTask{Url: seed, Depth: 1, Header: make(map[string]string)}
		ms.queue.Add(task)
	}

	// create crawlers, thread count was defined in conf
	ms.crawlers = make([]*Crawler, 0)
	for i := 0; i < conf.Basic.ThreadCount; i++ {
		crawler := NewCrawler(ms.urlTable, ms.config, &ms.queue)
		ms.crawlers = append(ms.crawlers, crawler)
	}

	return ms, nil
}

// run mini spider
func (ms *MiniSpider) Run() {
	// start all crawlers
	for _, crawler := range ms.crawlers {
		go crawler.Run()
	}
}

// get number of unfinished task
func (ms *MiniSpider) GetUnfinished() int {

	return ms.queue.GetUnfinished()
}<file_sep>/src/mini_spider/webpage_save_test.go
/* webpage_save_test.go - test for webpage_save.go  */
/*
modification history
--------------------
2017/07/22, by <NAME>, create
*/
/*
DESCRIPTION
*/
package mini_spider

import (
	"testing"
)

// test for genFilePath()
func TestGenFilePath(t *testing.T) {
	rootPath := "./output"
	url := "www.baidu.com"

	filePath := genFilePath(url, rootPath)
	if filePath != "./output/www.baidu.com" {
		t.Errorf("err in genFilePath(), filePath=%s", filePath)
	}
}

// test for saveWebPage()
func TestSaveWebPage(t *testing.T) {
	rootPath := "./output"
	url := "www.baidu.com"
	data := []byte("this is a test")

	err := saveWebPage(rootPath, url, data)
	if err != nil {
		t.Errorf("err in saveWebPage(%s, %s):%s", rootPath, url, err.Error())
	}
}

<file_sep>/src/web_package/webpage_save.go
/* webpage_save.go */
/*
modification history
--------------------
2017/07/21, by <NAME>, create
*/
/*
DESCRIPTION
*/
package web_package

import (
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"path"
)

const (
	OutputFileMode = 0644
)

/*
generate root directory to save web page

Params:
	- rootPath: root path for saving file
Returns:
	- error: any failure
*/
func genRootDir(rootPath string) error{
	if _, err := os.Stat(rootPath); os.IsNotExist(err) {
		if os.MkdirAll(rootPath, 0777) != nil {
			return fmt.Errorf("os.MkdirAll(%s):%s", rootPath, err.Error())
		}
	}

	return nil
}

/*
generate file path for given url

Params:
	- url: url to crawl
	- rootPath: root path for saving file

Returns:
	- file path
*/
func genFilePath(urlStr, rootPath string) string {
	filePath := url.QueryEscape(urlStr)
	filePath = path.Join(rootPath, filePath)
	return filePath
}

/*
save web page of given url to file

Params:
	- rootPath: root path for saving file
	- url: url to crawl
	- data: data to save
Returns:
	- error: any failure
*/
func SaveWebPage(rootPath string, url string, data []byte) error {
	// create root dir, if not exist
	if err := genRootDir(rootPath); err != nil {
		return fmt.Errorf("genRootDir(%s):%s", rootPath, err.Error())
	}

	// generate full file path
	filePath := genFilePath(url, rootPath)

	// save to file
	err := ioutil.WriteFile(filePath, data, OutputFileMode)
	if err != nil {
		return fmt.Errorf("ioutil.WriteFile(%s):%s", filePath, err.Error())
	}

	return nil
}
<file_sep>/src/mini_spider/mini_spider_test.go
/* mini_spider_test.go: test for mini_spider.go */
/*
modification history
--------------------
2017/07/21, by <NAME>, create
*/
/*
DESCRIPTION
*/

package mini_spider

import (
    "testing"
)

import (
    "mini_spider_config"
)

func TestNewMiniSpider(t *testing.T) {
    conf, _ := mini_spider_config.LoadConfig("../mini_spider_config/test_data/spider.conf")
    seeds, _ := LoadSeedFile(conf.Basic.UrlListFile)
    _, err := NewMiniSpider(&conf, seeds)
    if err != nil {
        t.Errorf("err happen in NewMiniSpider:%s", err.Error())
    }
}
<file_sep>/src/mini_spider/seedfile_load.go
/* seedfile_load.go: load seed file*/
/*
modification history
--------------------
2017/07/16, by <NAME>, create
*/
/*
DESCRIPTION
*/

package mini_spider

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
)

/*
load seed data from file

Params:
	- filePath: full path of seed file

Returns:
	- urls
	- error
*/
func LoadSeedFile(filePath string) ([]string, error) {
	// read json data from file
	data, err := ioutil.ReadFile(filePath)
	if err != nil {
		return nil, fmt.Errorf("ioutil.ReadFile(%s): %s", filePath, err.Error())
	}

	// decode json data
	var seeds []string
	err = json.Unmarshal(data, &seeds)
	if err != nil {
		return nil, fmt.Errorf("json.Unmarshal(): %s", err.Error())
	}

	return seeds, nil
}<file_sep>/src/mini_spider_config/conf_basic.go
/* conf_basic.go - basic config for mini spider */
/*
modification history
--------------------
2017/07/20, by linxiongmin, create
*/
/*
DESCRIPTION
*/

package mini_spider_config

import (
	"fmt"
	"regexp"
)

type BasicConfig struct {
	UrlListFile             string
	OutputDirectory         string
	MaxDepth                int
	CrawlInterval           int
	CrawlTimeout            int
	TargetUrl               string
	ThreadCount             int
	GracefulShutdownTimeout int
}

func (conf *BasicConfig) Check() error {
	// check urlListFile
	if conf.UrlListFile == "" {
		return fmt.Errorf("UrlListFile not set\n")
	}

	// check OutputDirectory
	if conf.OutputDirectory == "" {
		return fmt.Errorf("OutputDirectory not set\n")
	}

	// check MaxDepth
	if conf.MaxDepth < 1 {
		return fmt.Errorf("MaxDepth should >= 1\n")
	}

	// check CrawlInterval
	if conf.CrawlInterval < 1 {
		return fmt.Errorf("CrawlInterval should >= 1\n")
	}

	// check CrawlTimeout
	if conf.CrawlTimeout < 1 {
		return fmt.Errorf("CrawlTimeout should >= 1\n")
	}

	// check TargetUrl
	_, err := regexp.Compile(conf.TargetUrl)
	if err != nil {
		return fmt.Errorf("regexp.Compile(TargetUrl):%s", err.Error())
	}

	// check ThreadCount
	if conf.ThreadCount < 1 {
		return fmt.Errorf("ThreadCount should >= 1\n")
	}

	// check graceful shutdown timeout
	if conf.GracefulShutdownTimeout < 1 || conf.GracefulShutdownTimeout > 60{
		return fmt.Errorf("GracefulShutdownTimeout out of range [1, 60]\n")
	}

	return nil
}
<file_sep>/src/mini_spider/crawler.go
/* crawler.go - crawler thread*/
/*
modification history
--------------------
2017/07/18, by <NAME>, create
2017/08/01, by <NAME>, modify
 - add crawlChild function
*/
/*
DESCRIPTION
*/
package mini_spider

import (
	"fmt"
	"regexp"
	"time"
)

import (
	"www.baidu.com/golang-lib/http_util"
	"www.baidu.com/golang-lib/log"
)

import (
	"mini_spider_config"
	"web_package"
)

type Crawler struct {
	urlTable   *UrlTable
	config     *mini_spider_config.BasicConfig
	queue      *Queue
	urlPattern *regexp.Regexp
	stop       bool
}

// create new crawler
func NewCrawler(urlTable *UrlTable, config *mini_spider_config.MiniSpiderConf, queue *Queue) *Crawler {
	c := new(Crawler)
	c.urlTable = urlTable
	c.config = &config.Basic
	c.queue = queue

	// TargetUrl has been checked in conf load
	c.urlPattern, _ = regexp.Compile(c.config.TargetUrl)

	c.stop = false

	return c
}

// start crawler
func (c *Crawler) Run() {
	for !c.stop {
		// get new task from queue
		task := c.queue.Pop()
		log.Logger.Debug("from queue: url=%s, depth=%d", task.Url, task.Depth)

		// read data from given task
		data, err := http_util.Read(task.Url, c.config.CrawlTimeout, task.Header)
		if err != nil {
			log.Logger.Error("http_util.Read(%s):%s", task.Url, err.Error())
			c.queue.FinishOneTask()
			continue
		}

		// save data to file
		if c.urlPattern.MatchString(task.Url) {
			err = web_package.SaveWebPage(c.config.OutputDirectory, task.Url, data)
			if err != nil {
				log.Logger.Error("web_package.SaveWebPage(%s):%s", task.Url, err.Error())
			} else {
				log.Logger.Debug("save to file: %s", task.Url)
			}
		}

		// add to url table
		c.urlTable.Add(task.Url)

		// continue crawling until max depth
		if task.Depth < c.config.MaxDepth {
			err = c.crawlChild(data, task)
			if(err != nil){
				log.Logger.Error("crawlChild(%s):%s in depth of %d", task.Url, err.Error(), task.Depth)
			}
		}

		// confirm to remove task from queue
		c.queue.FinishOneTask()

		// sleep for a while
	    time.Sleep(time.Duration(c.config.CrawlInterval) * time.Second)
	}
}

// stop crawler
func (c *Crawler) Stop() {
	c.stop = true
}

// crawl child url
func (c *Crawler) crawlChild(data []byte, task *CrawlTask ) error {
	// parse url from web page
	links, err := web_package.ParseWebPage(data, task.Url)
	if err != nil {
		return fmt.Errorf("web_package.ParseWebPage():%s", err.Error())
	}

	// add child task to queue
	for _, link := range links {
		// check whether url match the pattern, or url exists already
		if c.urlTable.Exist(link) {
			continue
		}

		taskNew := &CrawlTask{Url: link, Depth: task.Depth + 1, Header: make(map[string]string)}
		log.Logger.Debug("add to queue: url=%s, depth=%d", taskNew.Url, taskNew.Depth)
		c.queue.Add(taskNew)
	}

	return nil
}
<file_sep>/start.sh
#!/bin/bash

cd output/bin
./mini_spider -c ../../conf/spider.conf -l ../log/ -s -d
<file_sep>/src/mini_spider/crawler_test.go
/* crawler_test.go: test for crawler */
/*
modification history
--------------------
2017/07/21, by <NAME>, create
*/
/*
DESCRIPTION
*/

package mini_spider

import (
    "sync"
    "testing"
)

import (
    "mini_spider_config"
)

func TestCrawler(t *testing.T) {
    urlTable := NewUrlTable()
    conf, _ := mini_spider_config.LoadConfig("../mini_spider_config/test_data/spider.conf")
    
    var queue Queue
    queue.Init()
    queue.Add(&CrawlTask{"http://pycm.baidu.com:8081", 2, nil})

    c := NewCrawler(urlTable, &conf, &queue)

    // wait and check result
    var wg sync.WaitGroup
    wg.Add(1)
    go func() {
        c.Run()
        wg.Done()
    }()
    wg.Wait()

    // check visit result
    verifyLinks := []string{
        "http://pycm.baidu.com:8081/page1.html",
        "http://pycm.baidu.com:8081/page2.html",
        "http://pycm.baidu.com:8081/page3.html",
        "http://pycm.baidu.com:8081/mirror/index.html",
        "http://pycm.baidu.com:8081/page4.html",
    }

    for _, link := range verifyLinks {
        if !c.urlTable.Exist(link) {
            t.Errorf("%s not visited", link)
        }
    }
}
<file_sep>/src/mini_spider_config/config.go
/* config.go*/
/*
modification history
--------------------
2017/07/18, by linxiongmin, create
*/
/*
DESCRIPTION
*/

package mini_spider_config

import (
	"fmt"
)

import (
	"code.google.com/p/gcfg"
)

type MiniSpiderConf struct {
	Basic  BasicConfig
}

// load config and check for validation
func LoadConfig(confPath string) (MiniSpiderConf, error) {
	conf, err := loadConfig(confPath)
	if err != nil {
		return conf, fmt.Errorf("configLoad(): %s\n", err.Error())
	}

	// check for validation
	if err = conf.Check(); err != nil {
		return conf, fmt.Errorf("conf.check(): %s\n", err.Error())
	}

	return conf, nil
}

func loadConfig(confPath string) (MiniSpiderConf, error) {
	var conf MiniSpiderConf
	var err error

	//load conf file to conf struct
	err = gcfg.ReadFileInto(&conf, confPath)
	if err != nil {
		return conf, fmt.Errorf("gcfg.ReadFileInto(): %s\n", err.Error())
	}

	return conf, nil
}

// check config validation
func (conf *MiniSpiderConf) Check() error {
	var err error

    // check conf data
	if err = conf.Basic.Check(); err != nil {
		return fmt.Errorf("Basic.Check(): %s", err.Error())
	}

	return nil
}
