tickserver/server/tick/future_sina_his.go

// const.go
package tick

import "fmt"
import "net/http"
import "encoding/json"
import "tickserver/server/market"
import "strconv"
import "io/ioutil"
import "time"

import "os"

import "errors"

const (
	RB0 = iota
	AG0
	AU0
	CU0
	AL0
	ZN0
	PB0
	RU0
	FU0
	WR0
	A0
	M0
	Y0
	J0
	C0
	L0
	P0
	V0
	RS0
	RM0
	FG0
	CF0
	WS0
	ER0
	ME0
	RO0
	TA0
	IH1505
	IC1505
)

var CtpHisNames []string

var SinaFutureSymbols = map[string]int{
	"RB0":    RB0,
	"AG0":    AG0,
	"AU0":    AU0,
	"CU0":    CU0,
	"AL0":    AL0,
	"ZN0":    ZN0,
	"PB0":    PB0,
	"RU0":    RU0,
	"FU0":    FU0,
	"WR0":    WR0,
	"A0":     A0,
	"M0":     M0,
	"Y0":     Y0,
	"J0":     J0,
	"C0":     C0,
	"L0":     L0,
	"P0":     P0,
	"V0":     V0,
	"RS0":    RS0,
	"RM0":    RM0,
	"FG0":    FG0,
	"CF0":    CF0,
	"WS0":    WS0,
	"ER0":    ER0,
	"ME0":    ME0,
	"RO0":    RO0,
	"TA0":    TA0,
	"IH1505": IH1505,
	"IC1505": IC1505,
}

var SinaFuturePeriods = []string{
	"5m",
	"60m",
	"1d",
}

func saveCtpHisNames() error {
	b, err := json.MarshalIndent(CtpHisNames, "", "  ")
	if err != nil {
		return err
	}
	ioutil.WriteFile("CtpHisNames.json", b, os.ModePerm)
	return nil
}

func LoadCtpHisNames() error {
	f, err := os.Open("CtpHisNames.json")
	if err != nil {
		return err
	}
	defer f.Close()
	dec := json.NewDecoder(f)
	err = dec.Decode(&CtpHisNames)
	if err != nil {
		return err
	}
	return nil
}

/*func DownloadCtp(db *market.MyDB, dataDir, typ, symbolComplex string) error {
	symbol := strings.Split(symbolComplex, ":")[0]
	bGoods, _ := strconv.Atoi(strings.Split(symbolComplex, ":")[1])
	var period int
	dir := dataDir + "/" + typ
	insId := fmt.Sprintf("ctp_%s", symbol)
	for _, v := range SinaFuturePeriods {
		if v == "5m" {
			period = market.M5
		}
		if v == "60m" {
			period = market.H1
		}
		if v == "1d" {
			period = market.D1
		}

		candles, err := downloadCtpPeriod(symbol, bGoods, v)
		if err != nil {
			log.Println(symbol, bGoods, v, err)
			return err
		}
		log.Println(symbol, bGoods, v, len(candles))
		//for ci, cv := range candles {
		//log.Println(ci, cv)
		//}

		//err = market.SaveCandlesAppend(db, dir, insId, candles, period)
		//if err != nil {
		//log.Println("SaveCandlesAppend", symbol, period, err)
		////break //return err
		//}
		if v == "5m" {
			//candlesm1 := candlem5ToCandlem1(candles)
			ticks := candleToTick(candles)
			//_, err = market.SaveTick(db, dir, ticks, insId, true)
			//if err != nil {
			//log.Println("SaveTick", symbol, err)
			//break //return err
			//}
			//err = market.ConvAndSaveCandles(db, dir, insId, ticks)
			//if err != nil {
			//log.Println("ConvAndSaveCandles", symbol, err)
			//break //return err
			//}
			//for ci, cv := range ticks {
			//log.Println(ci, cv)
			//}
			//_, err = market.SaveTickAppend(db, dir, ticks, insId)
			//if err != nil {
			//log.Println("SaveTickAppend", symbol, err)
			////break //return err
			//}
			candlesm1 := candlem5ToCandlem1(candles)
			//for ci, cv := range candlesm1 {
			//log.Println(ci, cv)
			//}
			//err = market.SaveCandlesAppend(db, dir, insId, candlesm1, market.M1)
			//if err != nil {
			//log.Println("SaveCandlesAppend", symbol, err)
			////break //return err
			//}
		}
	}

	return nil
}*/

/*func DownloadCtp(db *market.MyDB, dataDir, typ, symbolComplex string) error {
	symbol := strings.Split(symbolComplex, ":")[0]
	bGoods, _ := strconv.Atoi(strings.Split(symbolComplex, ":")[1])
	var period int
	dir := dataDir + "/" + typ
	insId := fmt.Sprintf("ctp_%s", symbol)
	for _, v := range SinaFuturePeriods {
		if v == "5m" {
			period = market.M5
		}
		if v == "60m" {
			period = market.H1
		}
		if v == "1d" {
			period = market.D1
		}

		candles, err := downloadCtpPeriod(symbol, bGoods, v)
		if err != nil {
			log.Println(symbol, bGoods, v, err)
			return err
		}
		log.Println(symbol, bGoods, v, len(candles))

		var key string
		candlesMap := make(map[string][]market.Candle)
		for _, candleV := range candles {
			t := time.Unix(candleV.Timestamp/1000, 0)
			if v == "5m" {
				key = fmt.Sprintf("%d%d%d", t.Year(), t.Month(), t.Day())
			}
			if v == "60m" {
				key = fmt.Sprintf("%d%d", t.Year(), t.Month())
			}
			if v == "1d" {
				key = fmt.Sprintf("%d", t.Year())
			}
			_, ok := candlesMap[key]
			if ok {
				candlesMap[key] = append(candlesMap[key], candleV)
			} else {
				var tmpCandles []market.Candle
				tmpCandles = append(tmpCandles, candleV)
				candlesMap[key] = tmpCandles
			}
		}
		for _, cs := range candlesMap {
			err = market.SaveCandlesAppend(db, dir, insId, cs, period)
			if err != nil {
				log.Println("SaveCandlesAppend", symbol, period, err)
				break //return err
			}
			if v == "5m" {
				//candlesm1 := candlem5ToCandlem1(candles)
				ticks := candleToTick(cs)
				//_, err = market.SaveTick(db, dir, ticks, insId, true)
				//if err != nil {
				//log.Println("SaveTick", symbol, err)
				//break //return err
				//}
				//err = market.ConvAndSaveCandles(db, dir, insId, ticks)
				//if err != nil {
				//log.Println("ConvAndSaveCandles", symbol, err)
				//break //return err
				//}
				_, err = market.SaveTickAppend(db, dir, ticks, insId)
				if err != nil {
					log.Println("SaveTickAppend", symbol, err)
					break //return err
				}
				candlesm1 := candlem5ToCandlem1(cs)
				err = market.SaveCandlesAppend(db, dir, insId, candlesm1, market.M1)
				if err != nil {
					log.Println("SaveCandlesAppend", symbol, err)
					break //return err
				}
			}
		}

		//for _, candle := range candles {
		//log.Println(candle)
		//}
	}

	return nil
}*/

/*func DownloadSinaFuture(db *market.MyDB, dataDir, typ, symbol string) error {
	var period int
	dir := dataDir + "/" + typ
	insId := fmt.Sprintf("sina_%s", symbol)
	for _, v := range SinaFuturePeriods {
		if v == "5m" {
			period = market.M5
		}
		if v == "60m" {
			period = market.H1
		}
		if v == "1d" {
			period = market.D1
		}

		var lastTime int64
		//row, err := db.GetLastTime(insId, period)
		//if err != nil {
		//log.Println("db.GetLastTime", err)
		////return err
		//}
		//if row != nil {
		//lastTime = row.StartTime
		//}

		candles, err := downloadSinaFuturePeriod(symbol, v, lastTime)
		if err != nil {
			return err
		}

		var key string
		candlesMap := make(map[string][]market.Candle)
		for _, candleV := range candles {
			t := time.Unix(candleV.Timestamp/1000, 0)
			if v == "5m" {
				key = fmt.Sprintf("%d%d%d", t.Year(), t.Month(), t.Day())
			}
			if v == "60m" {
				key = fmt.Sprintf("%d%d", t.Year(), t.Month())
			}
			if v == "1d" {
				key = fmt.Sprintf("%d", t.Year())
			}
			_, ok := candlesMap[key]
			if ok {
				candlesMap[key] = append(candlesMap[key], candleV)
			} else {
				var tmpCandles []market.Candle
				tmpCandles = append(tmpCandles, candleV)
				candlesMap[key] = tmpCandles
			}
		}
		for _, cs := range candlesMap {
			//err = market.SaveCandlesAppend(db, dir, insId, cs, period)
			//if err != nil {
			//log.Println("SaveCandlesAppend", symbol, period, err)
			//break //return err
			//}
			if v == "5m" {
				//candlesm1 := candlem5ToCandlem1(candles)
				ticks := candleToTick(cs)
				//_, err = market.SaveTick(db, dir, ticks, insId, true)
				//if err != nil {
				//log.Println("SaveTick", symbol, err)
				//break //return err
				//}
				//err = market.ConvAndSaveCandles(db, dir, insId, ticks)
				//if err != nil {
				//log.Println("ConvAndSaveCandles", symbol, err)
				//break //return err
				//}
				//_, err = market.SaveTickAppend(db, dir, ticks, insId)
				//if err != nil {
				//log.Println("SaveTickAppend", symbol, err)
				//break //return err
				//}
				candlesm1 := candlem5ToCandlem1(cs)
				//err = market.SaveCandlesAppend(db, dir, insId, candlesm1, market.M1)
				//if err != nil {
				//log.Println("SaveCandlesAppend", symbol, err)
				//break //return err
				//}
			}
		}

		//for _, candle := range candles {
		//log.Println(candle)
		//}
	}

	return nil
}*/

func downloadCtpPeriod(symbol string, bGoods int, period string) ([]market.Candle, error) {
	var u string
	if period == "1d" {
		if bGoods == 1 {
			u = fmt.Sprintf("http://stock2.finance.sina.com.cn/futures/api/json.php/IndexService.getInnerFuturesDailyKLine?symbol=%s", symbol)
		} else {
			u = fmt.Sprintf("http://stock2.finance.sina.com.cn/futures/api/json.php/CffexFuturesService.getCffexFuturesDailyKLine?symbol=%s", symbol)
		}
	} else {
		if bGoods == 1 {
			u = fmt.Sprintf("http://stock2.finance.sina.com.cn/futures/api/json.php/IndexService.getInnerFuturesMiniKLine%s?symbol=%s", period, symbol)
		} else {
			u = fmt.Sprintf("http://stock2.finance.sina.com.cn/futures/api/json.php/CffexFuturesService.getCffexFuturesMiniKLine%s?symbol=%s", period, symbol)
		}
	}
	res, err := http.Get(u)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}
	var candles [][6]string
	err = json.Unmarshal(b, &candles)
	if err != nil {
		return nil, err
	}
	if len(candles) == 0 {
		return nil, errors.New("no data")
	}
	candleLen := len(candles)
	candlelist := make([]market.Candle, candleLen)
	for i, v := range candles {
		var candle market.Candle
		var layout string
		if len(v[0]) == 10 {
			layout = "2006-01-02"
		} else {
			layout = "2006-01-02 15:04:05"
		}
		candleTime, err := time.Parse(layout, v[0])
		if err != nil {
			return nil, err
		}
		candle.Timestamp = candleTime.Unix() * 1000
		//if !checkTime("DCE", candle.Timestamp) {
		//continue
		//}
		candle.Open, err = strconv.ParseFloat(v[1], 64)
		if err != nil {
			return nil, err
		}
		candle.High, err = strconv.ParseFloat(v[2], 64)
		if err != nil {
			return nil, err
		}
		candle.Low, err = strconv.ParseFloat(v[3], 64)
		if err != nil {
			return nil, err
		}
		candle.Close, err = strconv.ParseFloat(v[4], 64)
		if err != nil {
			return nil, err
		}
		candle.RealVolums, err = strconv.ParseFloat(v[5], 64)
		if err != nil {
			return nil, err
		}
		if period == "1d" {
			candlelist[i] = candle
		} else {
			candlelist[candleLen-i-1] = candle
		}
	}
	return candlelist, nil
}

func downloadSinaFuturePeriod(symbol, period string, lastTime int64) ([]market.Candle, error) {
	var u string
	if period == "1d" {
		u = fmt.Sprintf("http://stock2.finance.sina.com.cn/futures/api/json.php/IndexService.getInnerFuturesDailyKLine?symbol=%s", symbol)
	} else {
		u = fmt.Sprintf("http://stock2.finance.sina.com.cn/futures/api/json.php/IndexService.getInnerFuturesMiniKLine%s?symbol=%s", period, symbol)
	}
	res, err := http.Get(u)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}
	var candles [][6]string
	err = json.Unmarshal(b, &candles)
	if err != nil {
		return nil, err
	}
	candleLen := len(candles)
	candlelist := make([]market.Candle, candleLen)
	for i, v := range candles {
		var candle market.Candle
		var layout string
		if len(v[0]) == 10 {
			layout = "2006-01-02"
		} else {
			layout = "2006-01-02 15:04:05"
		}
		candleTime, err := time.Parse(layout, v[0])
		if err != nil {
			return nil, err
		}
		candle.Timestamp = candleTime.Unix() * 1000
		if candle.Timestamp < lastTime {
			continue
		}
		//if !checkTime("DCE", candle.Timestamp) {
		//continue
		//}
		candle.Open, err = strconv.ParseFloat(v[1], 64)
		if err != nil {
			return nil, err
		}
		candle.High, err = strconv.ParseFloat(v[2], 64)
		if err != nil {
			return nil, err
		}
		candle.Low, err = strconv.ParseFloat(v[3], 64)
		if err != nil {
			return nil, err
		}
		candle.Close, err = strconv.ParseFloat(v[4], 64)
		if err != nil {
			return nil, err
		}
		candle.RealVolums, err = strconv.ParseFloat(v[5], 64)
		if err != nil {
			return nil, err
		}
		if period == "1d" {
			candlelist[i] = candle
		} else {
			candlelist[candleLen-i-1] = candle
		}
	}
	return candlelist, nil
}

func candleToTick(candles []market.Candle) []market.Tick {
	var ticks []market.Tick
	for i := 0; i < len(candles); i++ {
		ticko := market.Tick{}
		ticko.Price = candles[i].Open
		ticko.Timestamp = candles[i].Timestamp

		tickh := market.Tick{}
		tickh.Price = candles[i].High
		tickh.Timestamp = candles[i].Timestamp

		tickl := market.Tick{}
		tickl.Price = candles[i].Low
		tickl.Timestamp = candles[i].Timestamp

		tickc := market.Tick{}
		tickc.Price = candles[i].Close
		tickc.Timestamp = candles[i].Timestamp

		ticks = append(ticks, ticko)
		if candles[i].Open >= candles[i].Close {
			ticks = append(ticks, tickh)
			ticks = append(ticks, tickl)
		} else {
			ticks = append(ticks, tickl)
			ticks = append(ticks, tickh)
		}
		ticks = append(ticks, tickc)
		//log.Println(tickc)
	}
	return ticks
}

func candlem5ToCandlem1(candles []market.Candle) []market.Candle {
	var ticks []market.Candle
	for i := 0; i < len(candles); i++ {
		ticko := market.Candle{}
		ticko.Open = candles[i].Open
		ticko.High = candles[i].Open
		ticko.Low = candles[i].Open
		ticko.Close = candles[i].Open
		ticko.RealVolums = candles[i].RealVolums / 5
		ticko.Timestamp = candles[i].Timestamp

		tickh := market.Candle{}
		tickh.Open = candles[i].High
		tickh.High = candles[i].High
		tickh.Low = candles[i].High
		tickh.Close = candles[i].High
		tickh.RealVolums = candles[i].RealVolums / 5

		tickl := market.Candle{}
		tickl.Open = candles[i].Low
		tickl.High = candles[i].Low
		tickl.Low = candles[i].Low
		tickl.Close = candles[i].Low
		tickl.RealVolums = candles[i].RealVolums / 5

		tickc := market.Candle{}
		tickc.Open = candles[i].Close
		tickc.High = candles[i].Close
		tickc.Low = candles[i].Close
		tickc.Close = candles[i].Close
		tickc.RealVolums = candles[i].RealVolums / 5
		tickc.Timestamp = candles[i].Timestamp + 180000

		tickcc := tickc
		tickcc.Timestamp += 60000

		ticks = append(ticks, ticko)
		if candles[i].Open >= candles[i].Close {
			tickh.Timestamp = candles[i].Timestamp + 60000
			tickl.Timestamp = candles[i].Timestamp + 120000
			ticks = append(ticks, tickh)
			ticks = append(ticks, tickl)
		} else {
			tickl.Timestamp = candles[i].Timestamp + 60000
			tickh.Timestamp = candles[i].Timestamp + 120000
			ticks = append(ticks, tickl)
			ticks = append(ticks, tickh)
		}
		ticks = append(ticks, tickc)
		ticks = append(ticks, tickcc)
		//log.Println(tickc)
	}
	return ticks
}