私信发送成功
Watch Star Fork

fasiondog / hikyuuPythonMIT

基于C++/Python的开源量化交易研究框架 http://hikyuu.org/
克隆/下载
一键复制 编辑 原始数据 按行查看 历史
hdf5index.py 22.81 KB fasiondog 提交于 2015-01-07 01:26 . 初始版本
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720
#!/usr/bin/python
# -*- coding: utf8 -*-
# cp936
"""
建立HDF5索引
"""
import datetime
import tables
class IndexRecord(tables.IsDescription):
datetime = tables.UInt64Col() #IGNORE:E1101
start = tables.UInt64Col() #IGNORE:E1101
def UpdateWeekIndex(h5file):
try:
group = h5file.getNode("/","week")
except:
group = h5file.createGroup("/","week")
def getNewDate(olddate):
y = olddate/100000000
m = olddate/1000000 - y*100
d = olddate/10000 - (y*10000+m*100)
tempdate = datetime.date(y,m,d)
tempweekdate = tempdate - datetime.timedelta(tempdate.weekday())
newdate = tempweekdate.year*100000000 + tempweekdate.month*1000000 + tempweekdate.day*10000
return newdate
for table in h5file.walkNodes("/data"):
if type(table) != tables.table.Table:
continue
#print table.name
try:
index_table = h5file.getNode(group,table.name)
except:
index_table = h5file.createTable(group,table.name, IndexRecord)
total = table.nrows
if 0 == total:
continue
index_total = index_table.nrows
index_row = index_table.row
if index_total:
index_last_date = int(index_table[-1]['datetime'])
last_date = getNewDate(int(table[-1]['datetime']))
if index_last_date == last_date:
continue
startix = int(index_table[-1]['start'])
pre_index_date = int(index_table[-1]['datetime'])
else:
startix = 0
pre_index_date = getNewDate(int(table[0]['datetime']))
index_row['datetime'] = pre_index_date
index_row['start'] = 0
index_row.append()
#week_table.flush()
index = startix
for row in table[startix:]:
date = int(row['datetime'])
cur_index_date = getNewDate(date)
if cur_index_date != pre_index_date:
index_row['datetime'] = cur_index_date
index_row['start'] = index
index_row.append()
pre_index_date = cur_index_date
index += 1
index_table.flush()
def UpdateMonthIndex(h5file):
try:
group = h5file.getNode("/","month")
except:
group = h5file.createGroup("/","month")
def getNewDate(olddate):
y = olddate/100000000
m = olddate/1000000 - y*100
return(y*100000000 + m*1000000 + 10000)
for table in h5file.walkNodes("/data"):
if type(table) != tables.table.Table:
continue
#print table.name
try:
index_table = h5file.getNode(group,table.name)
except:
index_table = h5file.createTable(group,table.name, IndexRecord)
total = table.nrows
if 0 == total:
continue
index_total = index_table.nrows
index_row = index_table.row
if index_total:
index_last_date = int(index_table[-1]['datetime'])
last_date = getNewDate(int(table[-1]['datetime']))
if index_last_date == last_date:
continue
startix = int(index_table[-1]['start'])
pre_index_date = int(index_table[-1]['datetime'])
else:
startix = 0
date = int(table[0]['datetime'])
pre_index_date = getNewDate(date)
index_row['datetime'] = pre_index_date
index_row['start'] = 0
index_row.append()
#week_table.flush()
index = startix
for row in table[startix:]:
date = int(row['datetime'])
cur_index_date = getNewDate(date)
if cur_index_date != pre_index_date:
index_row['datetime'] = cur_index_date
index_row['start'] = index
index_row.append()
pre_index_date = cur_index_date
index += 1
index_table.flush()
def UpdateYearIndex(h5file):
try:
group = h5file.getNode("/","year")
except:
group = h5file.createGroup("/","year")
def getNewDate(olddate):
y = olddate/100000000
return(y*100000000 + 1010000)
for table in h5file.walkNodes("/data"):
if type(table) != tables.table.Table:
continue
#print table.name
try:
index_table = h5file.getNode(group,table.name)
except:
index_table = h5file.createTable(group,table.name, IndexRecord)
total = table.nrows
if 0 == total:
continue
index_total = index_table.nrows
index_row = index_table.row
if index_total:
index_last_date = int(index_table[-1]['datetime'])
last_date = getNewDate(int(table[-1]['datetime']))
if index_last_date == last_date:
continue
startix = int(index_table[-1]['start'])
pre_index_date = int(index_table[-1]['datetime'])
else:
startix = 0
date = int(table[0]['datetime'])
pre_index_date = getNewDate(date)
index_row['datetime'] = pre_index_date
index_row['start'] = 0
index_row.append()
#week_table.flush()
index = startix
for row in table[startix:]:
date = int(row['datetime'])
cur_index_date = getNewDate(date)
if cur_index_date != pre_index_date:
index_row['datetime'] = cur_index_date
index_row['start'] = index
index_row.append()
pre_index_date = cur_index_date
index += 1
index_table.flush()
def UpdateHalfYearIndex(h5file):
try:
group = h5file.getNode("/","halfyear")
except:
group = h5file.createGroup("/","halfyear")
def getNewDate(olddate):
halfyearDict={1:1,2:1,3:1,4:1,5:1,6:1,7:7,8:7,9:7,10:7,11:7,12:7}
y = olddate/100000000
m = olddate/1000000 - y*100
return( y*100000000 + halfyearDict[m]*1000000 + 10000 )
for table in h5file.walkNodes("/data"):
if type(table) != tables.table.Table:
continue
#print table.name
try:
index_table = h5file.getNode(group,table.name)
except:
index_table = h5file.createTable(group,table.name, IndexRecord)
total = table.nrows
if 0 == total:
continue
index_total = index_table.nrows
index_row = index_table.row
if index_total:
index_last_date = int(index_table[-1]['datetime'])
last_date = getNewDate(int(table[-1]['datetime']))
if index_last_date == last_date:
continue
startix = int(index_table[-1]['start'])
pre_index_date = int(index_table[-1]['datetime'])
else:
startix = 0
date = int(table[0]['datetime'])
pre_index_date = getNewDate(date)
index_row['datetime'] = pre_index_date
index_row['start'] = 0
index_row.append()
#week_table.flush()
index = startix
for row in table[startix:]:
date = int(row['datetime'])
cur_index_date = getNewDate(date)
if cur_index_date != pre_index_date:
index_row['datetime'] = cur_index_date
index_row['start'] = index
index_row.append()
pre_index_date = cur_index_date
index += 1
index_table.flush()
def UpdateQuarterIndex(h5file):
try:
group = h5file.getNode("/","quarter")
except:
group = h5file.createGroup("/","quarter")
def getNewDate(olddate):
quarterDict={1:1,2:1,3:1,4:4,5:4,6:4,7:7,8:7,9:7,10:10,11:10,12:10}
y = olddate/100000000
m = olddate/1000000 - y*100
return( y*100000000 + quarterDict[m]*1000000 + 10000 )
for table in h5file.walkNodes("/data"):
if type(table) != tables.table.Table:
continue
#print table.name
try:
index_table = h5file.getNode(group,table.name)
except:
index_table = h5file.createTable(group,table.name, IndexRecord)
total = table.nrows
if 0 == total:
continue
index_total = index_table.nrows
index_row = index_table.row
if index_total:
index_last_date = int(index_table[-1]['datetime'])
last_date = getNewDate(int(table[-1]['datetime']))
if index_last_date == last_date:
continue
startix = int(index_table[-1]['start'])
pre_index_date = int(index_table[-1]['datetime'])
else:
startix = 0
date = int(table[0]['datetime'])
pre_index_date = getNewDate(date)
index_row['datetime'] = pre_index_date
index_row['start'] = 0
index_row.append()
#week_table.flush()
index = startix
for row in table[startix:]:
date = int(row['datetime'])
cur_index_date = getNewDate(date)
if cur_index_date != pre_index_date:
index_row['datetime'] = cur_index_date
index_row['start'] = index
index_row.append()
pre_index_date = cur_index_date
index += 1
index_table.flush()
def UpdateDayIndex(h5file):
try:
group = h5file.getNode("/","day")
except:
group = h5file.createGroup("/","day")
def getNewDate(olddate):
newdate = olddate/10000*10000
return newdate
for table in h5file.walkNodes("/data"):
if type(table) != tables.table.Table:
continue
#print table.name
try:
index_table = h5file.getNode(group,table.name)
except:
index_table = h5file.createTable(group,table.name, IndexRecord)
total = table.nrows
if 0 == total:
continue
index_total = index_table.nrows
index_row = index_table.row
if index_total:
index_last_date = int(index_table[-1]['datetime'])
last_date = getNewDate(int(table[-1]['datetime']))
if index_last_date == last_date:
continue
startix = int(index_table[-1]['start'])
pre_index_date = int(index_table[-1]['datetime'])
else:
startix = 0
date = int(table[0]['datetime'])
pre_index_date = getNewDate(date)
index_row['datetime'] = pre_index_date
index_row['start'] = 0
index_row.append()
#week_table.flush()
index = startix
for row in table[startix:]:
date = int(row['datetime'])
cur_index_date = getNewDate(date)
if cur_index_date != pre_index_date:
index_row['datetime'] = cur_index_date
index_row['start'] = index
index_row.append()
pre_index_date = cur_index_date
index += 1
index_table.flush()
def UpdateHourIndex(h5file):
try:
group = h5file.getNode("/","min60")
except:
group = h5file.createGroup("/","min60")
def getNewDate(olddate):
min = olddate-olddate/10000*10000
if min<=1030:
newdate = olddate/10000*10000 + 1030
elif min<=1130:
newdate = olddate/10000*10000 + 1130
elif min<=1400:
newdate = olddate/10000*10000 + 1400
else:
newdate = olddate/10000*10000 + 1500
return newdate
for table in h5file.walkNodes("/data"):
if type(table) != tables.table.Table:
continue
#print table.name
try:
index_table = h5file.getNode(group,table.name)
except:
index_table = h5file.createTable(group,table.name, IndexRecord)
total = table.nrows
if 0 == total:
continue
index_total = index_table.nrows
index_row = index_table.row
if index_total:
index_last_date = int(index_table[-1]['datetime'])
last_date = getNewDate(int(table[-1]['datetime']))
if index_last_date == last_date:
continue
startix = int(index_table[-1]['start'])
pre_index_date = int(index_table[-1]['datetime'])
else:
startix = 0
date = int(table[0]['datetime'])
pre_index_date = getNewDate(date)
index_row['datetime'] = pre_index_date
index_row['start'] = 0
index_row.append()
index = startix
for row in table[startix:]:
date = int(row['datetime'])
cur_index_date = getNewDate(date)
if cur_index_date != pre_index_date:
index_row['datetime'] = cur_index_date
index_row['start'] = index
index_row.append()
pre_index_date = cur_index_date
index += 1
index_table.flush()
def UpdateFifteenMinIndex(h5file):
try:
group = h5file.getNode("/","min15")
except:
group = h5file.createGroup("/","min15")
def getNewDate(olddate):
min = olddate-olddate/10000*10000
if min<=945:
newdate = olddate/10000*10000 + 945
elif min<=1000:
newdate = olddate/10000*10000 + 1000
elif min<=1015:
newdate = olddate/10000*10000 + 1015
elif min<=1030:
newdate = olddate/10000*10000 + 1030
elif min<=1045:
newdate = olddate/10000*10000 + 1045
elif min<=1100:
newdate = olddate/10000*10000 + 1100
elif min<=1115:
newdate = olddate/10000*10000 + 1115
elif min<=1130:
newdate = olddate/10000*10000 + 1130
elif min<=1315:
newdate = olddate/10000*10000 + 1315
elif min<=1330:
newdate = olddate/10000*10000 + 1330
elif min<=1345:
newdate = olddate/10000*10000 + 1345
elif min<=1400:
newdate = olddate/10000*10000 + 1400
elif min<=1415:
newdate = olddate/10000*10000 + 1415
elif min<=1430:
newdate = olddate/10000*10000 + 1430
elif min<=1445:
newdate = olddate/10000*10000 + 1445
else:
newdate = olddate/10000*10000 + 1500
return newdate
for table in h5file.walkNodes("/data"):
if type(table) != tables.table.Table:
continue
#print table.name
try:
index_table = h5file.getNode(group,table.name)
except:
index_table = h5file.createTable(group,table.name, IndexRecord)
total = table.nrows
if 0 == total:
continue
index_total = index_table.nrows
index_row = index_table.row
if index_total:
index_last_date = int(index_table[-1]['datetime'])
last_date = getNewDate(int(table[-1]['datetime']))
if index_last_date == last_date:
continue
startix = int(index_table[-1]['start'])
pre_index_date = int(index_table[-1]['datetime'])
else:
startix = 0
date = int(table[0]['datetime'])
pre_index_date = getNewDate(date)
index_row['datetime'] = pre_index_date
index_row['start'] = 0
index_row.append()
index = startix
for row in table[startix:]:
date = int(row['datetime'])
cur_index_date = getNewDate(date)
if cur_index_date != pre_index_date:
index_row['datetime'] = cur_index_date
index_row['start'] = index
index_row.append()
pre_index_date = cur_index_date
index += 1
index_table.flush()
def UpdateHalfHourIndex(h5file):
try:
group = h5file.getNode("/","min30")
except:
group = h5file.createGroup("/","min30")
def getNewDate(olddate):
min = olddate-olddate/10000*10000
if min<=1000:
newdate = olddate/10000*10000 + 1000
elif min<=1030:
newdate = olddate/10000*10000 + 1030
elif min<=1100:
newdate = olddate/10000*10000 + 1100
elif min<=1130:
newdate = olddate/10000*10000 + 1130
elif min<=1330:
newdate = olddate/10000*10000 + 1330
elif min<=1400:
newdate = olddate/10000*10000 + 1400
elif min<=1430:
newdate = olddate/10000*10000 + 1430
else:
newdate = olddate/10000*10000 + 1500
return newdate
for table in h5file.walkNodes("/data"):
if type(table) != tables.table.Table:
continue
#print table.name
try:
index_table = h5file.getNode(group,table.name)
except:
index_table = h5file.createTable(group,table.name, IndexRecord)
total = table.nrows
if 0 == total:
continue
index_total = index_table.nrows
index_row = index_table.row
if index_total:
index_last_date = int(index_table[-1]['datetime'])
last_date = getNewDate(int(table[-1]['datetime']))
if index_last_date == last_date:
continue
startix = int(index_table[-1]['start'])
pre_index_date = int(index_table[-1]['datetime'])
else:
startix = 0
date = int(table[0]['datetime'])
pre_index_date = getNewDate(date)
index_row['datetime'] = pre_index_date
index_row['start'] = 0
index_row.append()
index = startix
for row in table[startix:]:
date = int(row['datetime'])
cur_index_date = getNewDate(date)
if cur_index_date != pre_index_date:
index_row['datetime'] = cur_index_date
index_row['start'] = index
index_row.append()
pre_index_date = cur_index_date
index += 1
index_table.flush()
def UpdateFiveMinIndex(h5file):
try:
group = h5file.getNode("/","min5")
except:
group = h5file.createGroup("/","min5")
def getNewDate(olddate):
newdate = olddate/100*100
min = olddate-newdate
if min == 0:
pass
elif min <= 5:
newdate += 5
elif min<=10:
newdate += 10
elif min<=15:
newdate += 15
elif min<=20:
newdate += 20
elif min<=25:
newdate += 25
elif min<=30:
newdate += 30
elif min<=35:
newdate += 35
elif min<=40:
newdate += 40
elif min<=45:
newdate += 45
elif min<=50:
newdate += 50
elif min<=55:
newdate += 55
else:
newdate += 100
return newdate
for table in h5file.walkNodes("/data"):
if type(table) != tables.table.Table:
continue
#print table.name
try:
index_table = h5file.getNode(group,table.name)
except:
index_table = h5file.createTable(group,table.name, IndexRecord)
total = table.nrows
if 0 == total:
continue
index_total = index_table.nrows
index_row = index_table.row
if index_total:
index_last_date = int(index_table[-1]['datetime'])
last_date = getNewDate(int(table[-1]['datetime']))
if index_last_date == last_date:
continue
startix = int(index_table[-1]['start'])
pre_index_date = int(index_table[-1]['datetime'])
else:
startix = 0
date = int(table[0]['datetime'])
pre_index_date = getNewDate(date)
index_row['datetime'] = pre_index_date
index_row['start'] = 0
index_row.append()
#week_table.flush()
index = startix
for row in table[startix:]:
date = int(row['datetime'])
cur_index_date = getNewDate(date)
if cur_index_date != pre_index_date:
index_row['datetime'] = cur_index_date
index_row['start'] = index
index_row.append()
pre_index_date = cur_index_date
index += 1
index_table.flush()
def UpdateDayDataAllIndex(h5file):
UpdateWeekIndex(h5file)
UpdateMonthIndex(h5file)
UpdateQuarterIndex(h5file)
UpdateHalfYearIndex(h5file)
UpdateYearIndex(h5file)
def Update5MinDataAllIndex(h5file):
UpdateFifteenMinIndex(h5file)
UpdateHalfHourIndex(h5file)
UpdateHourIndex(h5file)
#UpdateDayIndex(h5file)
#UpdateWeekIndex(h5file)
#UpdateMonthIndex(h5file)
#UpdateQuarterIndex(h5file)
#UpdateHalfYearIndex(h5file)
#UpdateYearIndex(h5file)
def Update1MinDataAllIndex(h5file):
UpdateFiveMinIndex(h5file)
#UpdateFifteenMinIndex(h5file)
#UpdateHalfHourIndex(h5file)
#UpdateHourIndex(h5file)
#UpdateDayIndex(h5file)
#UpdateWeekIndex(h5file)
#UpdateMonthIndex(h5file)
#UpdateQuarterIndex(h5file)
#UpdateHalfYearIndex(h5file)
#UpdateYearIndex(h5file)
if __name__ == "__main__":
import time
starttime = time.time()
print "\nUpdate SH Day Data index ================> "
h5file = tables.openFile('d:/workspace/hikyuu/test/data/sh_day.h5', mode='a',
filters=tables.Filters(complevel=9,complib='zlib', shuffle=True))
UpdateDayDataAllIndex(h5file)
h5file.close()
print "\nUpdate SH 5min Data index ================> "
h5file = tables.openFile('d:/workspace/hikyuu/test/data/sh_5min.h5', mode='a',
filters=tables.Filters(complevel=9,complib='zlib', shuffle=True))
Update5MinDataAllIndex(h5file)
h5file.close()
print "\nUpdate SZ Day Data index ================> "
h5file = tables.openFile('d:/workspace/hikyuu/test/data/sz_day.h5', mode='a',
filters=tables.Filters(complevel=9,complib='zlib', shuffle=True))
UpdateDayDataAllIndex(h5file)
h5file.close()
print "\nUpdate SZ 5min Data index ================> "
h5file = tables.openFile('d:/workspace/hikyuu/test/data/sz_5min.h5', mode='a',
filters=tables.Filters(complevel=9,complib='zlib', shuffle=True))
Update5MinDataAllIndex(h5file)
h5file.close()
endtime = time.time()
print "\nTotal time:"
print "%.2fs" % (endtime-starttime)
print "%.2fm" % ((endtime-starttime)/60)

评论 ( 0 )

你可以在登录后,对此项目发表评论

4_float_left_people 4_float_left_close