代码拉取完成,页面将自动刷新
'''
任务模块,负责封装各种启动模式,为多进程,单进程模式提供同一入口
'''
import asyncio
from concurrent import futures
import functools
import time
import sys
from log import log
import settings
from wikipedia import wikipedia
import statistics
from wordCount import wordCount
from wordTranslate import wordTranslate
import pdfCreator
logging = log(filename=settings.TASK_LOG_FILE, level=log.INFO)
tasks = []
def register(regTasks):
'''
任务注册函数
'''
for task in regTasks:
tasks.append(task)
def start():
'''
启动入口函数
'''
try:
to_do = []
taskPool = None
if settings.STARTMOD == settings.startMod.multiprocessing:
taskPool = futures.ProcessPoolExecutor
elif settings.STARTMOD == settings.startMod.singleprocessing:
taskPool = futures.ThreadPoolExecutor
else:
raise ValueError('start mod error {}'.format(settings.STARTMOD))
with taskPool(max_workers=len(tasks)) as executor:
for task in tasks:
func = task
future = executor.submit(func)
to_do.append(future)
statistics.syncShowstat() #显示统计在主线程/进程显示
except KeyboardInterrupt:
for future in to_do:
future.cancel()
except Exception as e:
logging.error("start error {}".format(e))
def asyncFuncPackage(func, cls=None):
'''
异步封装函数,async的函数可以通过此接口同步启动
'''
#print('run', func)
logging.info('run {}'.format(func.__name__))
try:
if cls:
a = cls()
asyncio.run(func(a))
else:
asyncio.run(func())
except Exception as e:
logging.error("async Func package error {}".format(e))
def funcPackage(func, cls):
'''
普通同步成员函数封装
'''
try:
a = cls()
func(a)
except Exception as e:
logging.error("funcPackage error {}".format(e))
def pack(packTasks):
'''
将Tasks列表打包成可以直接调用的无参数同步函数。
'''
try:
newTask = []
for task in packTasks:
if isinstance(task, tuple):
if asyncio.iscoroutinefunction(task[0]):
newTask.append(functools.partial(asyncFuncPackage, func=task[0], cls=task[1]))
else:
newTask.append(functools.partial(funcPackage, func=task[0], cls=task[1]))
elif asyncio.iscoroutinefunction(task):
newTask.append(functools.partial(asyncFuncPackage, func=task))
else:
newTask.append(task)
except Exception as e:
logging.error("pack error {}".format(e))
return newTask
def main(argv):
args = len(argv)
def showUsage():
print ("-----------------------------------")
print ("CLI usage: ")
print (" python3 task.py <start | create [number]>")
print ("example: start analyze")
print (" python3 task.py start")
print ("example: create pdf default(5000 words)")
print (" python3 task.py create ")
print ("example: create pdf top 1000 words)")
print (" python3 task.py create 1000")
print ("-----------------------------------")
print ("DOCKER usage: ")
print ("example: docker start")
print (" docker run -it -v ~/wordcountData:/root/code/data bingfengfeifei/wordcount start")
print ("example: docker create pdf")
print (" docker run -it -v ~/wordcountData:/root/code/data bingfengfeifei/wordcount create")
print ("-----------------------------------")
if args < 1 or args > 2:
showUsage()
return
elif argv[0] != 'start' and argv[0] != 'create':
showUsage()
return
elif argv[0] == 'start':
#直接加入该列表。目前不支持带参数的函数
#未打包的任务列表,如果是成员函数的话,放置在一个tuple里面,第一项是类名.方法名,第二项是类名。如果是非成员函数,
unpackTasks = [(wikipedia.initializeCategoryList, wikipedia), (wikipedia.startAnalyzeUrl, wikipedia),
(wikipedia.startAnalyzeHtml, wikipedia), (wordTranslate.start, wordTranslate),
(wordCount.syncCount, wordCount)]
startTasks = pack(unpackTasks)
register(startTasks)
start()
else:
try:
if args == 1:
num = 5000
else:
num = int(argv[1])
pdf = pdfCreator.pdfCreator()
pdf.generateWordList(num)
except Exception as e:
print(e)
showUsage()
return
if __name__ == '__main__':
main(sys.argv[1:])
此处可能存在不合适展示的内容,页面不予展示。您可通过相关编辑功能自查并修改。
如您确认内容无涉及 不当用语 / 纯广告导流 / 暴力 / 低俗色情 / 侵权 / 盗版 / 虚假 / 无价值内容或违法国家有关法律法规的内容,可点击提交进行申诉,我们将尽快为您处理。