From bc6062b7fb6b4a760d8569fdd6e609d1646029ed Mon Sep 17 00:00:00 2001 From: carson_git <543983849@qq.com> Date: Fri, 8 Jan 2021 14:33:09 +0800 Subject: [PATCH] done --- py/main.py | 190 ++-------------------- py/src/controllers/command_controller.py | 32 ++++ py/src/controllers/common.py | 23 +++ py/src/controllers/cusFuns_controller.py | 49 ++++++ py/src/controllers/groupby_controller.py | 20 +++ py/src/controllers/loaddata_controller.py | 31 ++++ py/src/controllers/query_controller.py | 14 ++ py/src/controllers/table_controller.py | 48 ++++++ py/src/core/Proxy.py | 7 + ui/src/api/index.ts | 8 +- 10 files changed, 246 insertions(+), 176 deletions(-) create mode 100644 py/src/controllers/command_controller.py create mode 100644 py/src/controllers/common.py create mode 100644 py/src/controllers/cusFuns_controller.py create mode 100644 py/src/controllers/groupby_controller.py create mode 100644 py/src/controllers/loaddata_controller.py create mode 100644 py/src/controllers/query_controller.py create mode 100644 py/src/controllers/table_controller.py diff --git a/py/main.py b/py/main.py index 888813d..62c6377 100644 --- a/py/main.py +++ b/py/main.py @@ -1,33 +1,39 @@ -from typing import Tuple + +from .src.controllers import ( + cusFuns_controller, query_controller, table_controller, + command_controller, groupby_controller, loaddata_controller) from flask import Flask import flask from flask import request -import pandas as pd -import numpy as np -import json import traceback import sys from .src.cusFuns.core.FunsPool import auto_register -from .src.helper.utils import json_converter -from .src.dataModel.pandasCmd import PandasCmd -from .src.core.Proxy import ProxyManager + import webbrowser import os import pathlib base_path = pathlib.Path(__file__).parent - -m_proxys = ProxyManager() auto_register(base_path, 'src/cusFuns') + app = Flask(__name__, template_folder='./web/templates', static_folder='./web/static') +app.register_blueprint(cusFuns_controller.app, url_prefix='/api/cus_fun') +app.register_blueprint(query_controller.app, url_prefix='/api/query') +app.register_blueprint(table_controller.app, url_prefix='/api/table') + +app.register_blueprint(command_controller.app, url_prefix='/api/cmd') +app.register_blueprint(groupby_controller.app, url_prefix='/api/groupby') +app.register_blueprint(loaddata_controller.app, url_prefix='/api/data_source') + + @app.errorhandler(Exception) def error_handler(ex): exc_type, exc_value, exc_traceback_obj = sys.exc_info() @@ -44,171 +50,11 @@ def error_handler(ex): } -def get_data_from_post(): - return json.loads(flask.request.get_data().decode()) - - @app.route('/') def index(): - # from . import __version__ - # pd.DataFrame({'a': [1, 2, 3]}).to_csv('test_ret.csv') - # return f'版本号：{__version__}' - return flask.render_template('index.html') -@app.route('/api/get_file_args/ext=', methods=['get']) -def get_file_args(ext: str): - print(ext) - print(request.values.get('ext')) - return 'done' - - -@app.route('/api/upload_file', methods=['post']) -def upload_file(): - - file = flask.request.files.get('file') - - ext = flask.request.form['ext'] - filename = flask.request.form['file_name'] - args = json.loads(flask.request.form['args']) - - # todo:根据不同情况做处理 - m_proxys.read_data(file, filename, ext, args) - return df2json(m_proxys.get_df_data()) - - -@app.route('/api/query', methods=['post']) -def query(): - param = get_data_from_post() - m_proxys.query(param['query']) - return df2json(m_proxys.get_df_data()) - - -@app.route('/api/table/filters', methods=['post']) -def table_filters(): - param = get_data_from_post() - m_proxys.query(param['query']) - return df2json(m_proxys.get_df_data()) - - -@app.route('/api/table/handle', methods=['post']) -def table_handle(): - - # todo:重构 - def _to_df_cols(df: pd.DataFrame, col: str): - # 如果是多层索引，使用eval转成元组 - if isinstance(df.columns, pd.MultiIndex): - return eval(col) - - return col - - param = get_data_from_post() - - filters = param['filters'] - sort = param['sort'] - - print(filters, sort) - - filters = ((f['field'], f['values']) for f in filters) - m_proxys.filter(filters) - - df_ret = m_proxys.get_df_data() - - if sort: - order = True if sort['order'] == 'asc' else False - df_ret = df_ret.sort_values(_to_df_cols( - df_ret, sort['field']), ascending=order) - - return df2json(df_ret) - - -@app.route('/api/groupby', methods=['post']) -def groupby(): - param = get_data_from_post() - keys = param['_keys'] - aggs = param['_aggs'] - - aggs = {d['filed']: d['methods'] for d in aggs} - - m_proxys.groupby(keys, aggs) - - return df2json(m_proxys.get_df_data()) - - -@app.route('/api/remove/cmd', methods=['post']) -def remove_cmd(): - param = get_data_from_post() - id = param['id'] - - m_proxys.remove_cmd(id) - return df2json(m_proxys.get_df_data()) - - -@app.route('/api/py_cody') -def get_py_code(): - ret = m_proxys.to_code() - return ret - - -@app.route('/api/cmds') -def get_cmds(): - cmds = m_proxys.get_cmds() - res = flask.json.dumps(cmds, default=json_converter) - return res - - -def df2json(df: pd.DataFrame, head_tail_num=10): - - ret = None - - if len(df) > (head_tail_num * 2): - ret = pd.concat([df.head(head_tail_num), df.tail(head_tail_num)]) - else: - ret = df - - cmd = PandasCmd.from_df(df, ret) - ret = flask.json.dumps(cmd, default=json_converter) - return ret - - -@app.route('/api/cus_fun', methods=['post']) -def cus_fun(): - param = get_data_from_post() - data = param['data'] - - m_proxys.run_cus_fun(data) - - return df2json(m_proxys.get_df_data()) - - -@app.route('/api/cus_fun/desc') -def get_cus_funcs_desc(): - # auto_register(base_path , 'src/cusFuns') - ret = m_proxys.get_cus_funcs_desc() - ret = flask.json.dumps(ret, default=json_converter) - return ret - - -@app.route('/api/cus_fun/model', methods=['get']) -def get_cus_funcs_ui_model(): - - uid = str(request.args.get('uid')) - - ret = m_proxys.get_ui_model(uid) - ret = flask.json.dumps(ret, default=json_converter) - return ret - - -@app.route('/api/cus_fun/input', methods=['post']) -def get_cus_funcs_input(): - param = get_data_from_post() - data = param['data'] - - m_proxys.input_cus_funcs(data) - return df2json(m_proxys.get_df_data()) - - def run(): port = 5551 @@ -244,9 +90,9 @@ def copy_bat(): def main(): args = sys.argv - has, dir = find_arg(args, '-dir') - if has: - os.chdir(dir) + # has, dir = find_arg(args, '-dir') + # if has: + # os.chdir(dir) has, _ = find_arg(args, '-c', return_value=False) if has: diff --git a/py/src/controllers/command_controller.py b/py/src/controllers/command_controller.py new file mode 100644 index 0000000..1af5578 --- /dev/null +++ b/py/src/controllers/command_controller.py @@ -0,0 +1,32 @@ + +from ..helper.utils import json_converter +import flask +from flask import request, Blueprint +import pandas as pd +from ..core.Proxy import ProxyManager +from . import common + +app = Blueprint('command', __name__) + + +@app.route('/remove', methods=['post']) +def remove_cmd(): + param = common.get_data_from_post() + id = param['id'] + + px = ProxyManager.get() + px.remove_cmd(id) + return common.df2json(px.get_df_data()) + + +@app.route('/') +def get_cmds(): + cmds = ProxyManager.get().get_cmds() + res = flask.json.dumps(cmds, default=json_converter) + return res + + +@app.route('/code') +def get_py_code(): + ret = ProxyManager.get().to_code() + return ret diff --git a/py/src/controllers/common.py b/py/src/controllers/common.py new file mode 100644 index 0000000..9bd5700 --- /dev/null +++ b/py/src/controllers/common.py @@ -0,0 +1,23 @@ +import json +from ..helper.utils import json_converter +from ..dataModel.pandasCmd import PandasCmd +import flask +import pandas as pd + + + +def df2json(df: pd.DataFrame, head_tail_num=10): + + ret = None + + if len(df) > (head_tail_num * 2): + ret = pd.concat([df.head(head_tail_num), df.tail(head_tail_num)]) + else: + ret = df + + cmd = PandasCmd.from_df(df, ret) + ret = flask.json.dumps(cmd, default=json_converter) + return ret + +def get_data_from_post(): + return json.loads(flask.request.get_data().decode()) diff --git a/py/src/controllers/cusFuns_controller.py b/py/src/controllers/cusFuns_controller.py new file mode 100644 index 0000000..39e7a6b --- /dev/null +++ b/py/src/controllers/cusFuns_controller.py @@ -0,0 +1,49 @@ + +import flask +from flask import request, Blueprint +from ..core.Proxy import ProxyManager +from ..helper.utils import json_converter +from . import common + +app = Blueprint('cus_fun', __name__) + + +@app.route('/', methods=['post']) +def cus_fun(): + param = common.get_data_from_post() + data = param['data'] + + px = ProxyManager.get() + + px.run_cus_fun(data) + + return common.df2json(px.get_df_data()) + + +@app.route('/desc') +def get_cus_funcs_desc(): + # auto_register(base_path , 'src/cusFuns') + ret = ProxyManager.get().get_cus_funcs_desc() + ret = flask.json.dumps(ret, default=json_converter) + return ret + + +@app.route('/model', methods=['get']) +def get_cus_funcs_ui_model(): + + uid = str(request.args.get('uid')) + + ret = ProxyManager.get().get_ui_model(uid) + ret = flask.json.dumps(ret, default=json_converter) + return ret + + +@app.route('/input', methods=['post']) +def get_cus_funcs_input(): + param = common.get_data_from_post() + data = param['data'] + + px = ProxyManager.get() + + px.input_cus_funcs(data) + return common.df2json(px.get_df_data()) diff --git a/py/src/controllers/groupby_controller.py b/py/src/controllers/groupby_controller.py new file mode 100644 index 0000000..5fbcba1 --- /dev/null +++ b/py/src/controllers/groupby_controller.py @@ -0,0 +1,20 @@ + +from flask import Blueprint +from ..core.Proxy import ProxyManager +from . import common + +app = Blueprint('groupby', __name__) + + +@app.route('/', methods=['post']) +def groupby(): + param = common.get_data_from_post() + keys = param['_keys'] + aggs = param['_aggs'] + + aggs = {d['filed']: d['methods'] for d in aggs} + + px = ProxyManager.get() + px.groupby(keys, aggs) + + return common.df2json(px.get_df_data()) diff --git a/py/src/controllers/loaddata_controller.py b/py/src/controllers/loaddata_controller.py new file mode 100644 index 0000000..8c2346b --- /dev/null +++ b/py/src/controllers/loaddata_controller.py @@ -0,0 +1,31 @@ + +import json +import flask +from flask import request, Blueprint +import pandas as pd +from ..core.Proxy import ProxyManager +from . import common + +app = Blueprint('loaddata', __name__) + + +# @app.route('/get_file_args/ext=', methods=['get']) +# def get_file_args(ext: str): +# print(ext) +# print(request.values.get('ext')) +# return 'done' + + +@app.route('/file', methods=['post']) +def upload_file(): + + file = flask.request.files.get('file') + + ext = flask.request.form['ext'] + filename = flask.request.form['file_name'] + args = json.loads(flask.request.form['args']) + + # todo:根据不同情况做处理 + px = ProxyManager.get() + px.read_data(file, filename, ext, args) + return common.df2json(px.get_df_data()) diff --git a/py/src/controllers/query_controller.py b/py/src/controllers/query_controller.py new file mode 100644 index 0000000..7a682f7 --- /dev/null +++ b/py/src/controllers/query_controller.py @@ -0,0 +1,14 @@ + +from flask import Blueprint +from ..core.Proxy import ProxyManager +from . import common + +app = Blueprint('query', __name__) + + +@app.route('/', methods=['post']) +def query(): + param = common.get_data_from_post() + px = ProxyManager.get() + px.query(param['query']) + return common.df2json(px.get_df_data()) diff --git a/py/src/controllers/table_controller.py b/py/src/controllers/table_controller.py new file mode 100644 index 0000000..44926de --- /dev/null +++ b/py/src/controllers/table_controller.py @@ -0,0 +1,48 @@ + +import flask +from flask import request, Blueprint +import pandas as pd +from ..core.Proxy import ProxyManager +from . import common + +app = Blueprint('table', __name__) + + +@app.route('/filters', methods=['post']) +def table_filters(): + param = common.get_data_from_post() + + px = ProxyManager.get() + px.query(param['query']) + return common.df2json(px.get_df_data()) + + +@app.route('/handle', methods=['post']) +def table_handle(): + + # todo:重构 + def _to_df_cols(df: pd.DataFrame, col: str): + # 如果是多层索引，使用eval转成元组 + if isinstance(df.columns, pd.MultiIndex): + return eval(col) + + return col + + param = common.get_data_from_post() + + filters = param['filters'] + sort = param['sort'] + + filters = ((f['field'], f['values']) for f in filters) + + px = ProxyManager.get() + px.filter(filters) + + df_ret = px.get_df_data() + + if sort: + order = True if sort['order'] == 'asc' else False + df_ret = df_ret.sort_values(_to_df_cols( + df_ret, sort['field']), ascending=order) + + return common.df2json(df_ret) diff --git a/py/src/core/Proxy.py b/py/src/core/Proxy.py index 91b04b6..0d5c6fa 100644 --- a/py/src/core/Proxy.py +++ b/py/src/core/Proxy.py @@ -98,6 +98,13 @@ class ProxyManager(object): 'xlsm': 'openpyxl' } + m_instance = None + + @staticmethod + def get(): + ProxyManager.m_instance = ProxyManager.m_instance or ProxyManager() + return ProxyManager.m_instance + def __init__(self) -> None: self.df_proxy = CallerProxy() self.pd_read_proxy = CallerProxy() diff --git a/ui/src/api/index.ts b/ui/src/api/index.ts index 6cfd754..17a566f 100644 --- a/ui/src/api/index.ts +++ b/ui/src/api/index.ts @@ -37,7 +37,7 @@ export async function uploadFile(file: UploadFile, args: FileArgsType) { headers: { "Content-Type": "multipart/form-data" }, }; - const ret = await axios.post('api/upload_file', param, config) + const ret = await axios.post('api/data_source/file', param, config) return ret.data as CmdResult // return ret.data } @@ -69,12 +69,12 @@ export async function query(queryString: string) { export async function getPyCode() { - const ret = await get('api/py_cody') + const ret = await get('api/cmd/code') return ret.data as string } export async function getAllCmds() { - const ret = await get('api/cmds') + const ret = await get('api/cmd') return ret.data as Command[] } @@ -85,7 +85,7 @@ export async function groupby(model: GroupbyModel) { } export async function removeCmd(id: string) { - const ret = await axios.post('/api/remove/cmd', { id }) + const ret = await axios.post('/api/cmd/remove', { id }) return ret.data as CmdResult } -- Gitee