From 9f6d4659bec15c4f8363f5dfa2663c90fb2ef6f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=80=E5=A7=8B=E8=AF=B4=E6=95=85=E4=BA=8B?= <7985556+fzgandw@user.noreply.gitee.com> Date: Tue, 22 Sep 2020 17:28:23 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9E=84=E9=80=A0=E6=95=B0=E6=8D=AE=E5=BA=93?= =?UTF-8?q?=E8=BF=9E=E6=8E=A5=E6=96=B9=E6=A1=88=20=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E9=93=BE=E6=8E=A5=E9=80=9A=E7=94=A8=E5=B7=A5=E5=85=B7?= =?UTF-8?q?=E3=80=82=20=E5=BD=93=E5=89=8D=E5=AE=9E=E7=8E=B0=E5=8A=9F?= =?UTF-8?q?=E8=83=BD=EF=BC=9A=20=EF=BC=881=EF=BC=89=E8=B4=A6=E5=8F=B7?= =?UTF-8?q?=E7=AE=A1=E7=90=86=20=EF=BC=882=EF=BC=89=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E8=BF=9E=E6=8E=A5=20=EF=BC=883=EF=BC=89SSH=E5=8A=A0?= =?UTF-8?q?=E5=AF=86=E9=80=9A=E9=81=93=20=EF=BC=884=EF=BC=89execute?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=EF=BC=9B=20=E8=BF=98=E9=9C=80=E5=AE=9E?= =?UTF-8?q?=E7=8E=B0=E5=8A=9F=E8=83=BD=EF=BC=9A=20=EF=BC=881=EF=BC=89?= =?UTF-8?q?=E5=86=99=E8=A1=A8=E5=8A=9F=E8=83=BD=20=EF=BC=882=EF=BC=89?= =?UTF-8?q?=E5=85=B6=E4=BB=96=E5=8A=9F=E8=83=BD=20=E7=9B=AE=E5=89=8D?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=EF=BC=8C=E5=8F=AF=E4=BB=A5=E8=BF=9E=E6=8E=A5?= =?UTF-8?q?=20MySQL=20=E5=92=8C=20PgSQL=E3=80=82=20=E8=BF=98=E9=9C=80?= =?UTF-8?q?=E8=A6=81=E4=B8=8E=E5=89=8D=E7=AB=AF=E8=BF=9B=E8=A1=8C=E5=AF=B9?= =?UTF-8?q?=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../__pycache__/dbBaseTool.cpython-37.pyc | Bin 0 -> 8254 bytes pyminer2/dbconnect/dbBaseTool.py | 237 ++++++++++++++++++ pyminer2/dbconnect/test_dbBaseTool.py | 62 +++++ 3 files changed, 299 insertions(+) create mode 100644 pyminer2/dbconnect/__pycache__/dbBaseTool.cpython-37.pyc create mode 100644 pyminer2/dbconnect/dbBaseTool.py create mode 100644 pyminer2/dbconnect/test_dbBaseTool.py diff --git a/pyminer2/dbconnect/__pycache__/dbBaseTool.cpython-37.pyc b/pyminer2/dbconnect/__pycache__/dbBaseTool.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f5212e7e1edb10e1ea8505d403e2270b7ff482f1 GIT binary patch literal 8254 zcmb7JTW}NC8Qxv3q?LTZ0f&$@Z8g&yY6C-{O;egsQV8jVI%!}^s-e}$I%_N=OU~I< z0LRXla47^EyA6Q^2$fq%a|6>REyi~0zICR3>GZAp8m%n9G@a>NpZfjh?5-_Yq^UG} z_UyUt|NYnV@3-69;tGEEOy|7uUZ!>I_`^n5fG6?Y?LJfW~KiyT*2#EcyarQ&#wG6TkJ7nhK@!Hh;)mP7A zSUiCr>)uf;6wIvsST+i_VxMPVu_6^Lt0O@)}oLdTVq&ejSv)l^ub z2)qS)z`KRDVuZokSUc{qoYJGRRjlKYz9?n2wMwG z2+Zm$H>+be+?kgbU;ZP|`)F=d0w@XoaOB1K($M$btR5ftxj>WY=bpOc&dyY3uelT8 z(CFgjk=h$aD|4^9Cr(vQ|E4xMn%G^;FppJYt*nX{pdZ~eTsv@C_oHCykdjt%3Q7ne zN==j@rhuGK*WlWX`x`0RIB+oUC`sLkqQbG0)4{s4Qa&pE_dq77P0aj`V}2+`n(e#@=kyzMZj55|NDqM;s%a z&J}WYI(;uuVW3d7h)Vy#PdC1-nB$MQ#>)HuP$V|z%GPa$`_U<4K+g)H}0<9JqF(m@Xzt9x&#yNom z(!3}xA%WXb8B)s1q%y1y0rPv6VGVZ;_s~#?h3-{Q>L@i97G*?~QBfNADm@W!DOUOd zmI;U>RIB3??)Z6FhRXb8_2^X4=Y-JaM13jMdyAIs=mSM=JBDTTrp2?UiMCd1D_JHd z@pZ)0K*q8T6*+@2-*v#W1t@`8OzHy4l(yqBCxp42q^c3l&K1+}PDu=ulB^pl{Rfjx zVQn@yfR==rgjztGKt)2U;@$_*?gOml5bpYzM?e&%m4n=V_bC(vFx@;nA4mX&#KQH{!P?2NvpNs! zQV%gF*%7kpgaC$=U`@d8w0lbAv9|$ueh+4D#$`oNC{aW0R0qHK$xsd;rm3Vk0%C@W z4{;6>eE4Ysx~xqqJC&d8+^-F5kYW{(YdIySPO7SM_<^C2*PlSYJ{*S4k!;b2!eyOd zlpOi&!~2wH*X@H%5_ae>*eXEgld_4_U&z0E>OyVu&5!0zc>s|(C){6*CYWhuKbl*@ zoL2r{{B*Bha8FEE%X97=JkzMOH-i1g!T#r^`P=&ZvaPm?eTIhv0-RM|FYV8B-DWb( z?*oN`xlTA|oBfs(DGe}qdj2VNBqNf;A#PfZdeBjyOKHy?#PwVQlbyR)Xvj|N@hH}? z+EEbObuhkNU9Be6!7qO@_R|u92&{0SV7kyS7$xY~qa+fcNqAb%gBD$5)czlk1f12C4+#)w?}JJN2{++xf5?xFP~l3 z-*kU<1u9a#^gilP3Tf(^u)_Vg5MawPXYSdd^(>dQDHibJWM3X~6f-`n$KSDqC1IA^+!vF#db@S;az9g!v-CZ}6r5q@fK7!1f>zBTt!lG$F~M#?5H-4Lmv|d6h;}-EzOK1SL#pYgLzOQ+>-lJ8SgGd0UTMJeYJMu zYGryRu`83eOtiqdE?k{KtR*Marmi)S8-L^K#TTm={@~uYvT*ihW#;6H9poAkEV(mS zSv`Nd`ra!|sUVSq^ORyn?83wHH@*Y1#4$hS3Ao^MjXGAqPCTOyKO!QY$cS`e><*g{ z`)Eq0WcQkdeiMN{qs;`M)i)=F4-xznJ9@zYjS#BPl-{o{Q!(Fxh?tpZcc*Sk#U;$f z$7}NxH@m__O0@v=0sj`>nt?ZifK;Y`7Z3nTM;^WlK1syLBH;KeT7+@rpGOVhvj~~# z0M+P#I;GpHH;O;PKpEb3wyN#gy3ifMyJm9($3wP}0(M~=DUh*++hOW4ZM<_>wKW;s z4XKkzBkV(B;fKo8$}@-{%3)y-=|rb>IohG_WW>>n17?9!IO9YHa@m7<)6rR}f53CW z*?iG5Ju*|_Lp`~?*+8fOf@8i6r{Gc7se|jA`qL)vBp@n;6#}AcK@^2hWY8y7Yp1Q- z2#V;W&UBQ>dW=O_6fVGEu`zvEWpUO*hCfF~w=hh9gh;icG@rEsnAO9YZIoj}+N2l( zSlT>Tv~pa)(gs-COaH{yfe4VM-YE{Q%$`k1z!KH-Kda3Cp)!44#>UdIN+T~#Qr&_w zu4)z`H-fh0FqWlPV8h&rQxN9HfGe0hD&zOhKET`NZB33x8nRR;KWy>a17r9J5kTx?kcTAM{WK!7|@D5 zjQ;*iZR!=$^Vcuna74!;#C5{r@>R6br>P~}f5dt=??i0VvYDOIcz0K_#nFluj%FFV z*NK>ibCzvMY1PS_%7}$jk*}wMj!%wGrv?5MRQ)6E*HBM|c{AyyaPtr@iz-UAUDH$z z352+~8(O@l$}mH@pQVs$Vz#tx2Dt2mNI#- zCz?*PVm6)TZ8VX>Q%)HTaZur(qZ%D-`4_1Ww2BycDFPEUI-+yLI0`Yv>O^5gbk&H) zV@AwqF;*Gz*87dH7?DM1@FQ!o9hbEUMbosH&caA+LFHH!_Xw#R?ok$p!n8PT0WgF} zi8v65dmxhVg76Sh4wq&8{;w3w?DCaKGe0)|5-AuzfXU|JP$=>W-u1Di{3UD^C2Ah< zgCt@~G9ZCAqMsKfh(YNR&j-nwk&CtYpC!-&GjJPjOcEJT9xOzHA>`hmGY>7{OQ%Q< zYzK9XmX*Ze94;9~Q;ubAKtNqStImoBNtGTiQcT=&Jiom`ArJyynhMWCGWPQ7H1(wb*hGU@Bg+rlOhFpjfr}zP; z>N!}euw`eE(Fq|1ms%w(o191ijSoIqDmi^gpG3CHOE4pbA{YKThFPRpiXKKOFlMsLaJ$%=~x!-#3OJqZ9^OumVc5hs) z+&*0&>IEm7y7#7cm(g?ecau2Z${2djp503UG7X_0-vXiqgMDA7I)+)u7Fn)vAQh$e z!{-cXt+iAH5@VO(Z*qm6q7ybbFY>NrR0_yWT8Ydh-;TnG^b~o2#)elCpCFvr|sL;u^%uP(K3!>rn642G=zcE}9ZZR$U%~ z#Cf?~$utyl5b_XAPO=V3_r!aP7p_-NkBBS*eUJ&d1#(@Wh2D!lMuqUH%@Y7I5h>D( z{Rtzb#e%8~-qY~@6i?EquB5@gNozhr1?6Qp9rL9N6iEMo9=oVm(#$1K_bqfx;Ic@m zj1W9s3_h-tQ+sp&mO#(P`!^V}s1s-1{%5Fdi{DD294}wQ}Np3{UjpQTJ?^<7|Pv=kB;l2psXgrmTN#&SoN9`Cz z;>VGUrxC}rW3W+Ul;Wh=95(85Xjm_-wZrAe0R=0`_NZ)!<$4&FD-O$5hUE&wa^2^T zWQN@`1kd7+6jm(!LO8HuTDg_1B)w<3P1NXnX1tUkT8IVyPx3~7K7_UL5?{Z%g5zdI z#G|!epIE$h(|rxO&{3!WzMjJ>0+}$9N>4>F;KlZU4Zqf8$aHKfjDg=3z5ivdxW;2w!DwLq!fxY7htHIS^qZ z#Og0t_*hvin5pO!ro9vG5RScA%xU*l@N5p>g~n9;ak(r$sPZS!(3NcGPvOqLOH-oW zlvE^J;#QI4L#D{Z3Pa{-11uGz9d>d0@}cJuf2WKd(?+D9&pV<1%mDu$twpjf%!AO_ zAWe&KKBZ@&?vBQKc!CD)reaB>h3xJ_+uv}BFOiB)-j&j`t?GSRJ3MR${_D{KbFz9R zT;%`nXh~@0WxFlek0;UbXWEh!LD-Klr?zC9utTXhUOkh~_L}`igw0RI5Lnpa`!Ww< zTq-g^AIUBL3?9U}gFl3t6DfB0kwYN^AQV}w+Uyk|4e~s&jowOFe 连接账号管理工具 + dbConnectTool -> 连接通道工具 + dbFuncTool -> SQL命令处理工具 + +""" +from sqlalchemy import create_engine +from sshtunnel import SSHTunnelForwarder +import pandas as pd +import pickle +import os + +class dbConnectAccountTool(object): + ''' + 数据库链接，账号处理。 + + 注意： + 创建以后，需要优先执行 LoadAccount + ''' + def __init__(self): + + self.pklroad = ".\dbConnectAccount.pkl" + self.dbconnectaccount = {} + self.dbtype = "" + self.connectname = "" + + def attachConnetName(self, dbtype = "", connectname = ""): + ''' + 参数： + 【1】 dbtype(str)：数据库类型 + 【2】 connectname(str)：链接的名称，用户填写，用于区分同一个数据库下的不同链接 + ''' + self.dbtype = dbtype + self.connectname = connectname + + def getConnectAccountSSH(self): + ''' + 提取SSH信息 + ''' + CA = self.getConnectAccount() + ssh = CA["SSH"] + account = CA["account"] + return(account["host"], account["port"], ssh["ssh_host"], ssh["ssh_port"], ssh["ssh_username"], ssh["ssh_password"]) + + def getConnectAccount(self): + ''' + 用途： + 获取连接 + 返回结果： + 连接的账号，IP，port，password, SSH 等 + ''' + self.loadConnectAccount() + connectaccount = self.dbconnectaccount[self.dbtype].get(self.connectname) + return(connectaccount) + + def getConnectAccountDesc(self): + ''' + 用途： + 获取连接的信息列表（connectname 和 desc） + 返回结果： + 字典格式，数据结构：{dbtype:{connectname: desc}} + ''' + self.loadConnectAccount() + res = {} + for k, v in self.dbconnectaccount.items(): + res.update({k: {}}) + for vk, vv in v.items(): + res[k].update({vk: vv.get("connectdescribe")}) + return(res) + + def delConnectAccount(self): + ''' + 用途： + 删除链接通道 + 注意： + 前端需要验证是否有选择需要删除的链接 + ''' + del self.dbconnectaccount[self.dbtype][self.connectname] + self.writeConnectAccount() + + def updateConnectAccount(self, connectaccount={}): + ''' + 用途： + 新增（更新）连接通道 + 参数： + 【3】 connectaccount(dict)：账号的信息，包括用户，密码，地址等，对应的JSON结构： + 注意：上游传入的 connectaccount 结构说明： + 【1】account(json) -> 账号信息 + |- （1）user = 用户 （2）password = 密码 （3）host = 地址 + |- （4）port = 端口 （5）database = 数据库 （6）charset = 字符集 + + 【2】usessh(boolean) -> 是否使用 SSH 加密通道，默认为 False，表示不使用该通道 + 暂时实现了 SSH 加密通道使用密码加密的方法 + + 【3】SSH(dict) -> SSH加密通道 + |- （1）ssh_host = 地址 （2）ssh_port = 端口 （3）ssh_username = 用户名 + |- （4）ssh_authenmethod = 加密模式（5）ssh_password = 密码 + + 【4】 connectdescribe(str) -> 对连接的描述，前端传入时默认为空字符串 + ''' + if self.dbtype not in self.dbconnectaccount: + self.dbconnectaccount.update(dbtype={}) + + self.dbconnectaccount[self.dbtype].update({self.connectname: connectaccount}) + # 如果连接不存在，则直接新增； + # 前端需要检验并提醒是否有同名连接，如果同名会覆盖 + self.writeConnectAccount() + + def writeConnectAccount(self): + with open(self.pklroad, 'wb') as wfile: + pickle.dump(self.dbconnectaccount, wfile) + wfile.close() + + def loadConnectAccount(self): + ''' + 用途： + 从 pickle 文件中获取链接账号数据 + 返回： + dbConnectAccount(dict)：连接账号集合，数据结构：{数据库类型:{名称: {连接账号信息}}} + ''' + if not os.path.exists(self.pklroad): + # 如果文件不存在，则创建一个测试用账号写入到 pkl 文件中 + testdt = dict( + account = dict( + user="root", password="", host="localhost", + port="3306", database="", charset="utf-8" + ), + usessh = False, + SSH = {}, + connectdescribe = "这是一个测试模块" + ) + self.dbconnectaccount = {"mysql": {"testdt": testdt}} + self.writeConnectAccount() + + with open(self.pklroad, 'rb') as rfile: + self.dbconnectaccount = pickle.load(rfile) + rfile.close() + +class dbConnectTool(object): + ''' + 数据库通用连接工具 + ''' + def __init__(self, account, conn_url): + ''' + 参数： + 【1】 account(class)：dbConnectAccountTool(dbtype, connectname) + 【2】 conn_url(str)：通过 url 方式连接数据库 + ''' + self.account = account + self.conn_url = conn_url + + def createSSHConn(self): + ''' + 开启 SSH 连接方式 + ''' + ssh = self.account["SSH"] + account = self.account["account"] + self.ssh_server = SSHTunnelForwarder( + (ssh["ssh_host"], ssh["ssh_port"]), + ssh_username=ssh["ssh_username"], + ssh_password=ssh["ssh_password"], + remote_bind_address=(account["host"], account["port"]) + ) + self.ssh_server.start() + self.account["port"] = str(self.ssh_server.local_bind_port) + + def createConn(self): + ''' + 用途： + 通过 url 的方式创建连接通道 + 参数： + conn_url：连接url，由数据类型进行定义 + + 注意： + 暂时没有实现SSL的方法 + ''' + try: + if self.account["usessh"]: + self.createSSHConn() + self.engine = create_engine(self.conn_url.format(**self.account["account"]), encoding = "utf-8") + conn_status = {"status":"connect", "info":""} + except Exception as e: + conn_status = {"status":"error", "info":e} + + return(conn_status) + + def closeConn(self): + ''' + 关闭所有通道 + ''' + if self.account["usessh"]: + self.ssh_server.close() + +class dbFuncTool(object): + ''' + 数据库通用执行方法 + ''' + def __init__(self, account, conn_url): + self.dbCT = dbConnectTool(account = account, conn_url = conn_url) + self.conn_status = self.dbCT.createConn() + + def execute(self, sql): + ''' + 执行命令，需要增加一个装饰器，关于运行时间的装饰器 + 返回结果： + 字典结构，包含内容： + （1）data(pd.dataframe)：数据 + （2）execute_status(str)：查询结果状态，done = 正常；error = 报错 + （3）info(str)：返回信息。GetData = 返回数据，需要呈现；ExecuteSQL = 执行命令，不用呈现 + ''' + try: + conn = self.dbCT.engine.execute(sql) + if conn.cursor.description: + df = pd.DataFrame( + data = list(conn.cursor.fetchall()), + columns = list(map(lambda x:x[0], conn.cursor.description)) + ) + res = {"data":df, "execute_status":"done", "info":"GetData"} + else: + df = pd.DataFrame([]) + res = {"data":df, "execute_status":"done", "info":"ExecuteSQL"} + except Exception as e: + res = {"data":pd.DataFrame([]), "execute_status":"error", "info":str(e)} + conn.close() + self.dbCT.closeConn() + return(res) + + + # "mysql": "mysql+pymysql://{user}:{password}@{host}:{port}/{database}" + # "pgsql": "postgresql://{user}:{password}@{host}:{port}/{database}" + +# if __name__ == "__main__": diff --git a/pyminer2/dbconnect/test_dbBaseTool.py b/pyminer2/dbconnect/test_dbBaseTool.py new file mode 100644 index 00000000..22ccfacb --- /dev/null +++ b/pyminer2/dbconnect/test_dbBaseTool.py @@ -0,0 +1,62 @@ +import os +from dbBaseTool import * + +def split_print(dt): + print(dt) + print("=" * 50) + +dbCA = dbConnectAccountTool() +dbCA.pklroad = ".\dbConnectAccount.pkl" + +# 模拟打开数据库模块后，选择需要连接的方案： +desc = dbCA.getConnectAccountDesc() +split_print(desc) + +# 模拟获取某个账号的信息、SSH账号等信息 +dbtype, connectname = "mysql", "testdt" +dbCA.attachConnetName(dbtype, connectname) +account = dbCA.getConnectAccount() +split_print(account) + +# 模拟增加（或更新）一个新的连接 +dbtype, connectname = "mysql", "testaccount" +dbCA.attachConnetName(dbtype, connectname) +connectaccount = dict( + account=dict( + user="gandw", password="123456", host="localhost", + port="3306", database="local_db", charset="utf-8" + ), + usessh=False, + SSH={}, + connectdescribe="这又是一个测试" +) +dbCA.updateConnectAccount(connectaccount) +dbCA.loadConnectAccount() +split_print(dbCA.dbconnectaccount) + +# 模拟数据库的 "测试连接" +mysql_url = "mysql+pymysql://{user}:{password}@{host}:{port}/{database}" +dbtype, connectname = "mysql", "testaccount" +dbCA.attachConnetName(dbtype, connectname) +account = dbCA.getConnectAccount() +dbCT = dbConnectTool(account = account, conn_url = mysql_url) +connect_status = dbCT.createConn() +split_print(connect_status) + +# 模拟一个查询操作(mysql) +mysql_url = "mysql+pymysql://{user}:{password}@{host}:{port}/{database}" +dbtype, connectname = "mysql", "testaccount" +dbCA.attachConnetName(dbtype, connectname) +account = dbCA.getConnectAccount() +mysql = dbFuncTool(account = account, conn_url = mysql_url) +print(mysql.dbCT.engine) +df = mysql.execute(sql = "select * from mysql.use") +print(df['execute_status']) +print(df['data']) + +# 模拟删除一个连接 +dbtype, connectname = "mysql", "testaccount" +dbCA.attachConnetName(dbtype, connectname) +dbCA.delConnectAccount() +dbCA.loadConnectAccount() +# split_print(dbCA.dbconnectaccount) -- Gitee