tickserver/server/tick/ds_sina.go

package tick

import "net/http"
import "io/ioutil"
import "regexp"
import "strings"
import "bytes"
import "errors"
import "fmt"
import "strconv"
import "time"
import "log"
import "golang.org/x/text/encoding/simplifiedchinese"

var symbolurl = "http://quote.eastmoney.com/stock_list.html"
var lastMarket map[string]string

func GetInstrument() ([]string, []string, error) {
	return getInstrument()
}

func getInstrument() ([]string, []string, error) {
	resp, err := http.Get(symbolurl)
	if err != nil {
		return nil, nil, err
	}
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, nil, err
	}
	symbolcontent := bytes.SplitN(data, []byte(`<div id="quotesearch">`), 2)
	if len(symbolcontent) != 2 {
		return nil, nil, errors.New("no symbolcontent-0")
	}
	symbolcontent = bytes.SplitN(symbolcontent[1], []byte("</ul>"), 3)
	if len(symbolcontent) != 3 {
		return nil, nil, errors.New("no symbolcontent-1")
	}
	//sh symbol list
	shlist, err := getli(symbolcontent[0])
	if err != nil {
		return nil, nil, err
	}
	//sz
	szlist, err := getli(symbolcontent[1])
	if err != nil {
		return nil, nil, err
	}
	var sh []string
	for i := 0; i < len(shlist); i++ {
		sym , _ := getsymbol(string(shlist[i]))
		sh = append(sh, sym)
	}
	var sz []string
	for i := 0; i < len(szlist); i++ {
		sym, _ := getsymbol(string(szlist[i]))
		sz = append(sz, sym)
	}
	return sh, sz, nil
}

func GetInstrument2() ([]string, []string, []string, []string, error) {
	resp, err := http.Get(symbolurl)
	if err != nil {
		return nil, nil,nil, nil, err
	}
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, nil, nil, nil, err
	}
	symbolcontent := bytes.SplitN(data, []byte(`<div id="quotesearch">`), 2)
	if len(symbolcontent) != 2 {
		return nil, nil, nil, nil, errors.New("no symbolcontent-0")
	}
	symbolcontent = bytes.SplitN(symbolcontent[1], []byte("</ul>"), 3)
	if len(symbolcontent) != 3 {
		return nil, nil, nil, nil, errors.New("no symbolcontent-1")
	}
	//sh symbol list
	shlist, err := getli(symbolcontent[0])
	if err != nil {
		return nil, nil, nil, nil, err
	}
	//sz
	szlist, err := getli(symbolcontent[1])
	if err != nil {
		return nil, nil, nil ,nil, err
	}
	var sh []string
	var shname []string
    trans := simplifiedchinese.GBK.NewDecoder()
	dst := make([]byte, 1024)
	
	for i := 0; i < len(shlist); i++ {
		symbol, name := getsymbol(string(shlist[i]))
		sh = append(sh, symbol)
		nDst, _, err := trans.Transform(dst, []byte(name), true)
		if err == nil {
			name = string(dst[0:nDst])
		}
		shname = append(shname, name)
	}
	var sz []string
	var szname []string
	for i := 0; i < len(szlist); i++ {
		symbol, name := getsymbol(string(szlist[i]))
		sz = append(sz, symbol)
		nDst, _, err := trans.Transform(dst, []byte(name), true)
		if err == nil {
			name = string(dst[0:nDst])
		}
		szname = append(szname, name)
	}
	return sh, sz, shname, szname, nil
}

func getli(data []byte) ([][]byte, error) {
	return matchTag(data, "li")
}

func getsymbol(data string) (string, string) {
	data = stripTags(data)
	o, _, err := strRange(data, "(", ")", 0)
	if err != nil {
		return "", ""
	}
	name := strings.Replace(data, "("+o+")", "", -1)
	return o, name
}

func strRange(html string, start string, end string, offset int) (string, int, error) {
	istart := strings.SplitN(html, start, 2)
	if len(istart) != 2 {
		return "", 0, errors.New("start string not found")
	}
	if offset > 0 {
		istart[1] = istart[1][offset:]
	}
	iend := strings.SplitN(istart[1], end, 2)
	if len(iend) != 2 {
		return "", 0, errors.New("end string not found")
	}
	return iend[0], len(istart[0]) + len(iend[0]), nil
}

func matchTag(html []byte, tagname string) ([][]byte, error) {
	exp, err := regexp.Compile("(?i)<" + tagname + "[^>]*>(.*?)</" + tagname + ">")
	if err != nil {
		return nil, err
	}
	matched := exp.FindAllSubmatch(html, -1)
	if len(matched) == 0 {
		return nil, fmt.Errorf("tagname=%s not matched", tagname)
	}
	ret := make([][]byte, len(matched))
	for i := 0; i < len(ret); i++ {
		ret[i] = matched[i][1]
	}
	return ret, nil
}

var tagsRegexp = regexp.MustCompile("<[^>]+>")

func stripTags(html string) string {
	return tagsRegexp.ReplaceAllString(html, "")
}

type SinaDS struct {
	*DSBase
}

func init() {
	drivers[Sina] = newSinaDS
}

func newSinaDS(conf *DsConf) (DataSource, error) {
	return &SinaDS{
		DSBase: NewDsBase(conf), // lmax 自己下载历史数据, 所以参数db==nil
	}, nil
}

func (lds *SinaDS) Name() string {
	return Sina
}

func (lds *SinaDS) Run() {
	if !lds.conf.Run {
		log.Println("SinaDS.run config NOT run")
		return
	}
	log.Println("SinaDS.run")
	fn := func() {
		var urls []string
		for {
			sh, sz, err := getInstrument()
			if err != nil {
				time.Sleep(time.Second)
				continue
			}
			urls = getfetchurls(sh, sz)
			break
		}
		lastMarket = make(map[string]string)
		datach := make(chan []string, 20)
		// 登录产生新的Session
		for i := 0; i < len(urls); i++ {
			go func(n int) {
				for {
					fetchurl(IntSina, urls[n], datach)
					time.Sleep(time.Millisecond * 200)
				}
			}(i)
		}
		for {
			item := <-datach
			newdata := getnewdata(IntSina, item)
			for i := 0; i < len(newdata); i++ {
				lds.Save(newdata[i])
			}
		}
	}
	fn()
}

func getfetchurls(sh []string, sz []string) []string {
	baseurl := "http://hq.sinajs.cn/list="
	for i := 0; i < len(sh); i++ {
		sh[i] = "sh" + sh[i]
	}
	for i := 0; i < len(sz); i++ {
		sz[i] = "sz" + sz[i]
	}
	sh = append(sh, sz...)
	var ret []string
	for i := 0; i < len(sh); i = i + 200 {
		end := 200
		if i+200 > len(sh) {
			end = len(sh) - i
		}
		data := sh[i : i+end]
		query := strings.Join(data, ",")
		ret = append(ret, baseurl+query)
	}
	return ret
}

func fetchurl(typ int, url string, ch chan []string) {
	if typ == IntSinaFuture {
		//log.Println("sinafuture fetchurl begin")
	}
	resp, err := http.Get(url)
	if err != nil {
		ch <- nil
		log.Println("fetchurl get", err)
		return
	}
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		ch <- nil
		log.Println("fetchurl readall", err)
		return
	}
	tickdatas := parsetick(string(data))
	ch <- tickdatas
	if typ == IntSinaFuture {
		//log.Println("sinafuture fetchurl end")
	}
}

func parsetick(data string) []string {
	return strings.Split(data, "\n")
}

func getnewdata(typ int, datas []string) []*Market {
	var ret []*Market
	for i := 0; i < len(datas); i++ {
		datas[i] = strings.Replace(datas[i], "var hq_str_", "", -1)
		marketdata := strings.SplitN(datas[i], "=", 2)
		if len(marketdata) != 2 {
			continue
		}
		if lastdata, ok := lastMarket[marketdata[0]]; ok {
			if lastdata == marketdata[1] {
				if typ == IntSinaFuture {
					//log.Println("sinafuture same")
				}
				continue
			}
		}
		lastMarket[marketdata[0]] = marketdata[1]
		var market *Market
		var err error
		if typ == IntSina {
			market, err = parsemarket(marketdata[0], marketdata[1])
		} else {
			market, err = parsemarketfuture(marketdata[0], marketdata[1])
		}
		if err != nil {
			log.Println(datas[i], err)
			continue
		}
		ret = append(ret, market)
	}
	return ret
}

func parsemarket(name string, data string) (*Market, error) {
	data = strings.Trim(data, "\";\r\n")
	datas := strings.Split(data, ",")
	if len(datas) < 32 {
		return nil, errors.New("makret data error")
	}
	var market Market
	var err error
	market.Open, err = strconv.ParseFloat(datas[1], 64)
	if err != nil {
		return nil, err
	}
	market.LastPrice, err = strconv.ParseFloat(datas[2], 64)
	if err != nil {
		return nil, err
	}
	market.Close, err = strconv.ParseFloat(datas[3], 64)
	if err != nil {
		return nil, err
	}
	market.High, err = strconv.ParseFloat(datas[4], 64)
	if err != nil {
		return nil, err
	}
	market.Low, err = strconv.ParseFloat(datas[5], 64)
	if err != nil {
		return nil, err
	}
	market.AllVolume, err = strconv.ParseFloat(datas[8], 64)
	if err != nil {
		return nil, err
	}
	market.AllAmount, err = strconv.ParseFloat(datas[9], 64)
	if err != nil {
		return nil, err
	}
	bids := make([]PP, 5)
	bids[0][1], err = strconv.ParseFloat(datas[10], 64)
	if err != nil {
		return nil, err
	}
	bids[0][0], err = strconv.ParseFloat(datas[11], 64)
	if err != nil {
		return nil, err
	}
	bids[1][1], err = strconv.ParseFloat(datas[12], 64)
	if err != nil {
		return nil, err
	}
	bids[1][0], err = strconv.ParseFloat(datas[13], 64)
	if err != nil {
		return nil, err
	}
	bids[2][1], err = strconv.ParseFloat(datas[14], 64)
	if err != nil {
		return nil, err
	}
	bids[2][0], err = strconv.ParseFloat(datas[15], 64)
	if err != nil {
		return nil, err
	}
	bids[3][1], err = strconv.ParseFloat(datas[16], 64)
	if err != nil {
		return nil, err
	}
	bids[3][0], err = strconv.ParseFloat(datas[17], 64)
	if err != nil {
		return nil, err
	}
	bids[4][1], err = strconv.ParseFloat(datas[18], 64)
	if err != nil {
		return nil, err
	}
	bids[4][0], err = strconv.ParseFloat(datas[19], 64)
	if err != nil {
		return nil, err
	}
	market.Bids = bids
	asks := make([]PP, 5)
	asks[0][1], err = strconv.ParseFloat(datas[20], 64)
	if err != nil {
		return nil, err
	}
	asks[0][0], err = strconv.ParseFloat(datas[21], 64)
	if err != nil {
		return nil, err
	}
	asks[1][1], err = strconv.ParseFloat(datas[22], 64)
	if err != nil {
		return nil, err
	}
	asks[1][0], err = strconv.ParseFloat(datas[23], 64)
	if err != nil {
		return nil, err
	}
	asks[2][1], err = strconv.ParseFloat(datas[24], 64)
	if err != nil {
		return nil, err
	}
	asks[2][0], err = strconv.ParseFloat(datas[25], 64)
	if err != nil {
		return nil, err
	}
	asks[3][1], err = strconv.ParseFloat(datas[26], 64)
	if err != nil {
		return nil, err
	}
	asks[3][0], err = strconv.ParseFloat(datas[27], 64)
	if err != nil {
		return nil, err
	}
	asks[4][1], err = strconv.ParseFloat(datas[28], 64)
	if err != nil {
		return nil, err
	}
	asks[4][0], err = strconv.ParseFloat(datas[29], 64)
	if err != nil {
		return nil, err
	}
	market.Asks = asks
	t, err := time.Parse(time.RFC3339, datas[30]+"T"+datas[31]+"+08:00")
	if err != nil {
		return nil, err
	}
	market.Timestamp = t.Unix() * 1000
	name = name[2:]
	market.InsId, err = strconv.ParseInt(name, 10, 64)
	if err != nil {
		return nil, err
	}
	market.Type = IntSina
	return &market, nil
}