3 Star 8 Fork 1

冰封飞飞 / 计算机英语词频统计

加入 Gitee
与超过 1200万 开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)
免费加入
克隆/下载
task.py 4.92 KB
一键复制 编辑 原始数据 按行查看 历史
冰封飞飞 提交于 2018-11-12 22:29 . 修改提示信息
'''
任务模块,负责封装各种启动模式,为多进程,单进程模式提供同一入口
'''
import asyncio
from concurrent import futures
import functools
import time
import sys
from log import log
import settings
from wikipedia import wikipedia
import statistics
from wordCount import wordCount
from wordTranslate import wordTranslate
import pdfCreator
logging = log(filename=settings.TASK_LOG_FILE, level=log.INFO)
tasks = []
def register(regTasks):
'''
任务注册函数
'''
for task in regTasks:
tasks.append(task)
def start():
'''
启动入口函数
'''
try:
to_do = []
taskPool = None
if settings.STARTMOD == settings.startMod.multiprocessing:
taskPool = futures.ProcessPoolExecutor
elif settings.STARTMOD == settings.startMod.singleprocessing:
taskPool = futures.ThreadPoolExecutor
else:
raise ValueError('start mod error {}'.format(settings.STARTMOD))
with taskPool(max_workers=len(tasks)) as executor:
for task in tasks:
func = task
future = executor.submit(func)
to_do.append(future)
statistics.syncShowstat() #显示统计在主线程/进程显示
except KeyboardInterrupt:
for future in to_do:
future.cancel()
except Exception as e:
logging.error("start error {}".format(e))
def asyncFuncPackage(func, cls=None):
'''
异步封装函数,async的函数可以通过此接口同步启动
'''
#print('run', func)
logging.info('run {}'.format(func.__name__))
try:
if cls:
a = cls()
asyncio.run(func(a))
else:
asyncio.run(func())
except Exception as e:
logging.error("async Func package error {}".format(e))
def funcPackage(func, cls):
'''
普通同步成员函数封装
'''
try:
a = cls()
func(a)
except Exception as e:
logging.error("funcPackage error {}".format(e))
def pack(packTasks):
'''
将Tasks列表打包成可以直接调用的无参数同步函数。
'''
try:
newTask = []
for task in packTasks:
if isinstance(task, tuple):
if asyncio.iscoroutinefunction(task[0]):
newTask.append(functools.partial(asyncFuncPackage, func=task[0], cls=task[1]))
else:
newTask.append(functools.partial(funcPackage, func=task[0], cls=task[1]))
elif asyncio.iscoroutinefunction(task):
newTask.append(functools.partial(asyncFuncPackage, func=task))
else:
newTask.append(task)
except Exception as e:
logging.error("pack error {}".format(e))
return newTask
def main(argv):
args = len(argv)
def showUsage():
print ("-----------------------------------")
print ("CLI usage: ")
print (" python3 task.py <start | create [number]>")
print ("example: start analyze")
print (" python3 task.py start")
print ("example: create pdf default(5000 words)")
print (" python3 task.py create ")
print ("example: create pdf top 1000 words)")
print (" python3 task.py create 1000")
print ("-----------------------------------")
print ("DOCKER usage: ")
print ("example: docker start")
print (" docker run -it -v ~/wordcountData:/root/code/data bingfengfeifei/wordcount start")
print ("example: docker create pdf")
print (" docker run -it -v ~/wordcountData:/root/code/data bingfengfeifei/wordcount create")
print ("-----------------------------------")
if args < 1 or args > 2:
showUsage()
return
elif argv[0] != 'start' and argv[0] != 'create':
showUsage()
return
elif argv[0] == 'start':
#直接加入该列表。目前不支持带参数的函数
#未打包的任务列表,如果是成员函数的话,放置在一个tuple里面,第一项是类名.方法名,第二项是类名。如果是非成员函数,
unpackTasks = [(wikipedia.initializeCategoryList, wikipedia), (wikipedia.startAnalyzeUrl, wikipedia),
(wikipedia.startAnalyzeHtml, wikipedia), (wordTranslate.start, wordTranslate),
(wordCount.syncCount, wordCount)]
startTasks = pack(unpackTasks)
register(startTasks)
start()
else:
try:
if args == 1:
num = 5000
else:
num = int(argv[1])
pdf = pdfCreator.pdfCreator()
pdf.generateWordList(num)
except Exception as e:
print(e)
showUsage()
return
if __name__ == '__main__':
main(sys.argv[1:])
Python
1
https://gitee.com/bingfengfeifei/wordCount.git
git@gitee.com:bingfengfeifei/wordCount.git
bingfengfeifei
wordCount
计算机英语词频统计
master

搜索帮助