From d0770b535f211dcc4468bd4da76f807082e7aa0f Mon Sep 17 00:00:00 2001 From: "tanglinchuanz@163.com" Date: Mon, 7 Mar 2022 19:55:52 +0800 Subject: [PATCH 1/4] mod --- wxBackendsapi/efficient_crawler.py | 2 +- wxBackendsapi/models.py | 2 +- wxBackendsapi/views.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/wxBackendsapi/efficient_crawler.py b/wxBackendsapi/efficient_crawler.py index 5fc0379..b5ac539 100644 --- a/wxBackendsapi/efficient_crawler.py +++ b/wxBackendsapi/efficient_crawler.py @@ -224,7 +224,7 @@ def CrawlerF(year, month, session, sesskey): if event['normalisedeventtype'] == "course": course_info = event['course'] # shortname - shortname = course_info['shortname'][:199] + shortname = course_info['shortname'] shortname = cs.courseAbbr(shortname) data.append(shortname) # print(shortname) diff --git a/wxBackendsapi/models.py b/wxBackendsapi/models.py index acb3014..81fdc2a 100644 --- a/wxBackendsapi/models.py +++ b/wxBackendsapi/models.py @@ -21,7 +21,7 @@ class DDLInfo(models.Model): openid = models.CharField("openid", max_length=100, default="") # username = models.CharField("Username", max_length=20, default="") # pwd = models.CharField("pwd", max_length=100, default="") - courseName = models.CharField("Course Name", max_length=200) + courseName = models.CharField("Course Name", max_length=100) DDL_Name = models.CharField("DDL Name", max_length=200) DDL_Time = models.CharField("DDL Time", max_length=100) # courseState = models.BooleanField("Course State", default=True) diff --git a/wxBackendsapi/views.py b/wxBackendsapi/views.py index f3ef325..2374aff 100644 --- a/wxBackendsapi/views.py +++ b/wxBackendsapi/views.py @@ -359,6 +359,6 @@ def worker(openid): try: saveDDLInfo(openid, False) except BaseException as e: - logger.error('crawler failed on openid {} '.format(openid) + str(e)) + logger.error('crawler failed on openid {} '.format(openid) + e.args) finally: sema.release() \ No newline at end of file -- Gitee From f0f572a1ad96ff87baefdada2ad2c7ce7deec415 Mon Sep 17 00:00:00 2001 From: "tanglinchuanz@163.com" Date: Tue, 8 Mar 2022 08:49:03 +0800 Subject: [PATCH 2/4] mod logger --- wxBackendsapi/views.py | 7 ++++--- wxcloudrun/settings.py | 2 +- wxcloudrun/views.py | 10 +++++----- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/wxBackendsapi/views.py b/wxBackendsapi/views.py index 2374aff..61c5758 100644 --- a/wxBackendsapi/views.py +++ b/wxBackendsapi/views.py @@ -48,6 +48,7 @@ def saveDDLInfo(openid, isLogin): dict2 = we.CrawlerF(year+(month+1)//13, month%12+1, session, sesskey) data = {**dict1, **dict2} + logger.info('data: '+str(data)) for ddlg in queryset: if ddlg.eventID in data: @@ -113,8 +114,8 @@ def handCrawler(request): try: saveDDLInfo(rsp['openid'], True) except BaseException as e: - logger.error('crawler failed at openid: {} '.format(rsp['openid'])+str(e)) - return JsonResponse({'errcode': -6, 'errmsg': str(e)}, json_dumps_params={'ensure_ascii': False}) + logger.error('crawler failed at openid: {} '.format(rsp['openid'])+str(e.args)) + return JsonResponse({'errcode': -6, 'errmsg': str(e.args)}, json_dumps_params={'ensure_ascii': False}) return HttpResponse(json.dumps({"status": 200, "msg": "OK!", "IsUpdate":True}, ensure_ascii=False)) @@ -359,6 +360,6 @@ def worker(openid): try: saveDDLInfo(openid, False) except BaseException as e: - logger.error('crawler failed on openid {} '.format(openid) + e.args) + logger.error('crawler failed on openid {} '.format(openid) + str(e.args)) finally: sema.release() \ No newline at end of file diff --git a/wxcloudrun/settings.py b/wxcloudrun/settings.py index db6e56a..a79665d 100644 --- a/wxcloudrun/settings.py +++ b/wxcloudrun/settings.py @@ -192,7 +192,7 @@ LOGGING = { } CRONJOBS = [ - ('30 * * * *', 'wxBackendsapi.views.timer', ['',''], {}, '>> /var/log/crontab.log') + ('30 * * * *', 'wxBackendsapi.views.timer', ['',''], {}, '>> '+os.path.join(LOG_PATH, 'all-{}.log'.format(time.strftime('%Y-%m-%d')))) ] # Internationalization diff --git a/wxcloudrun/views.py b/wxcloudrun/views.py index 041408a..b6396f4 100644 --- a/wxcloudrun/views.py +++ b/wxcloudrun/views.py @@ -24,7 +24,7 @@ def tokenValidate(request): if user.isLogin == False: raise Exception("session expired", user.openid) except BaseException as e: - ret = {'errcode': -1, 'errorMsg': str(e)} + ret = {'errcode': -1, 'errorMsg': str(e.args)} else: ret = {'openid': token['openid']} else: @@ -103,7 +103,7 @@ def uicAuth(username, password): try: rsp = session.get(loginUrl, verify=False) except BaseException as e: - logger.error('request result: '+str(e)) + logger.error('request result: '+str(e.args)) else: loginToken = BeautifulSoup(rsp.text, 'html.parser').find("input",{'name': 'logintoken'})['value'] @@ -121,7 +121,7 @@ def uicAuth(username, password): try: rsp = session.post(loginUrl, postdata, headers=header, verify=False) except BaseException as e: - logger.error('request result: '+str(e)) + logger.error('request result: '+str(e.args)) else: sesskey = BeautifulSoup(rsp.text, 'html.parser').find("input",{'name': 'sesskey'})['value'] session = requests.utils.dict_from_cookiejar(session.cookies)['MoodleSession'] @@ -144,7 +144,7 @@ def weixinAuth(js_code): try: rsp = requests.get(url, params=payload, verify=False) except BaseException as e: - logger.error('request result: '+str(e)) + logger.error('request result: '+str(e.args)) else: logger.info('weixinAuth result: {}'.format(rsp.content.decode('utf-8'))) finally: @@ -231,6 +231,6 @@ def sessionExtend(openid): user = models.UserInfo.objects.get(openid=openid) user.isLogin = False user.save() - logger.error("session extend failed on openid: {} ".format(openid) + str(e)) + logger.error("session extend failed on openid: {} ".format(openid) + str(e.args)) return session, user.sesskey -- Gitee From 376ed48b1e47772543b8b266e3f3bc3c9e72b716 Mon Sep 17 00:00:00 2001 From: "tanglinchuanz@163.com" Date: Fri, 11 Mar 2022 12:33:27 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=88=AC=E8=99=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- wxBackendsapi/efficient_crawler.py | 2 +- wxBackendsapi/views.py | 124 +++++++++++++++-------------- wxcloudrun/views.py | 1 + 3 files changed, 67 insertions(+), 60 deletions(-) diff --git a/wxBackendsapi/efficient_crawler.py b/wxBackendsapi/efficient_crawler.py index b5ac539..acf56ac 100644 --- a/wxBackendsapi/efficient_crawler.py +++ b/wxBackendsapi/efficient_crawler.py @@ -192,7 +192,7 @@ def CrawlerF(year, month, session, sesskey): days = week['days'] for day in days: if day['popovertitle'] != '': - ddlTime = str(day['year']) + ', ' + day['popovertitle'].replace(' events', '').replace(',', '') + # ddlTime = str(day['year']) + ', ' + day['popovertitle'].replace(' events', '').replace(',', '') # print(ddlTime) if day['events'] != []: """ diff --git a/wxBackendsapi/views.py b/wxBackendsapi/views.py index 61c5758..f4acb29 100644 --- a/wxBackendsapi/views.py +++ b/wxBackendsapi/views.py @@ -37,67 +37,68 @@ import wxBackendsapi.courseTimetableCrawler as wcourse def saveDDLInfo(openid, isLogin): session, sesskey = vi.sessionExtend(openid) - # get current month - today = datetime.datetime.today() - year = today.year - month = today.month - - queryset = models.DDLInfo.objects.filter(openid=openid) - - dict1 = we.CrawlerF(year, month, session, sesskey) - dict2 = we.CrawlerF(year+(month+1)//13, month%12+1, session, sesskey) - - data = {**dict1, **dict2} - logger.info('data: '+str(data)) - - for ddlg in queryset: - if ddlg.eventID in data: - if ddlg.isShow == True: - # update ddl - ddl = data[ddlg.eventID] - if ddlg.DDL_Name != ddl[2]: - toTime = datetime.datetime.fromtimestamp(float(ddl[3])+8*3600).strftime('%m-%d %H:%M') - ddlg.DDL_Name = ddl[2] - ddlg.assState = True - if isLogin == False: - vi.notificationSend(os.environ.get('NOTIF_TEMPID_RELE'), openid, 'pages/list/list', ddl[1], ddl[2], toTime) - - if ddlg.DDL_Time != str(ddl[3]): - fromTime = datetime.datetime.fromtimestamp(float(ddlg.DDL_Time)+8*3600).strftime('%m-%d %H:%M') - toTime = datetime.datetime.fromtimestamp(float(ddl[3])+8*3600).strftime('%m-%d %H:%M') - ddlg.DDL_Time = ddl[3] - ddlg.assState = True - if isLogin == False: - vi.notificationSend(os.environ.get('NOTIF_TEMPID_RELE'), openid, 'pages/list/list', '时间更新'+ddl[1], ddl[2], fromTime+'~'+toTime) - - if str(ddlg.isSubmit) != str(ddl[4]): - ddlg.isSubmit = ddl[4] - - # ddl reminder - if str(ddl[4]) == 'False': - now = datetime.datetime.now().timestamp() - to = float(ddl[3]) - 3600*2 # 提前2小时 - if now + 3600 + 600 >= to and now <= to: # 检测范围为1h10m + if session != '': + # get current month + today = datetime.datetime.today() + year = today.year + month = today.month + + queryset = models.DDLInfo.objects.filter(openid=openid) + + dict1 = we.CrawlerF(year, month, session, sesskey) + dict2 = we.CrawlerF(year+(month+1)//13, month%12+1, session, sesskey) + + data = {**dict1, **dict2} + logger.info('dict: '+str(data)) + + for ddlg in queryset: + if ddlg.eventID in data: + if ddlg.isShow == True: + # update ddl + ddl = data[ddlg.eventID] + if ddlg.DDL_Name != ddl[2]: + toTime = datetime.datetime.fromtimestamp(float(ddl[3])+8*3600).strftime('%m-%d %H:%M') + ddlg.DDL_Name = ddl[2] ddlg.assState = True + if isLogin == False: + vi.notificationSend(os.environ.get('NOTIF_TEMPID_RELE'), openid, 'pages/list/list', ddl[1], ddl[2], toTime) + + if ddlg.DDL_Time != str(ddl[3]): + fromTime = datetime.datetime.fromtimestamp(float(ddlg.DDL_Time)+8*3600).strftime('%m-%d %H:%M') toTime = datetime.datetime.fromtimestamp(float(ddl[3])+8*3600).strftime('%m-%d %H:%M') + ddlg.DDL_Time = ddl[3] + ddlg.assState = True if isLogin == False: - vi.notificationSend(os.environ.get('NOTIF_TEMPID_URGE'), openid, 'pages/list/list', ddl[1], ddl[2], toTime) - + vi.notificationSend(os.environ.get('NOTIF_TEMPID_RELE'), openid, 'pages/list/list', '时间更新'+ddl[1], ddl[2], fromTime+'~'+toTime) + + if str(ddlg.isSubmit) != str(ddl[4]): + ddlg.isSubmit = ddl[4] + + # ddl reminder + if str(ddl[4]) == 'False': + now = datetime.datetime.now().timestamp() + to = float(ddl[3]) - 3600*2 # 提前2小时 + if now + 3600 + 600 >= to and now <= to: # 检测范围为1h10m + ddlg.assState = True + toTime = datetime.datetime.fromtimestamp(float(ddl[3])+8*3600).strftime('%m-%d %H:%M') + if isLogin == False: + vi.notificationSend(os.environ.get('NOTIF_TEMPID_URGE'), openid, 'pages/list/list', ddl[1], ddl[2], toTime) + + ddlg.save() + + del data[ddlg.eventID] + else: + # set isShow False + ddlg.isShow = False ddlg.save() - del data[ddlg.eventID] - else: - # set isShow False - ddlg.isShow = False - ddlg.save() + for ddl in data.values(): + # insert ddl + DDLInfo(eventID=ddl[0],openid=openid,courseName=ddl[1],DDL_Name=ddl[2],DDL_Time=ddl[3],isSubmit=ddl[4],moduleName=ddl[5]).save() + if isLogin == False: + vi.notificationSend(os.environ.get('NOTIF_TEMPID_RELE'), openid, 'pages/list/list', ddl[1], ddl[2], datetime.datetime.fromtimestamp(float(ddl[3])+8*3600).strftime('%m-%d %H:%M')) - for ddl in data.values(): - # insert ddl - DDLInfo(eventID=ddl[0],openid=openid,courseName=ddl[1],DDL_Name=ddl[2],DDL_Time=ddl[3],isSubmit=ddl[4],moduleName=ddl[5]).save() - if isLogin == False: - vi.notificationSend(os.environ.get('NOTIF_TEMPID_RELE'), openid, 'pages/list/list', ddl[1], ddl[2], datetime.datetime.fromtimestamp(float(ddl[3])+8*3600).strftime('%m-%d %H:%M')) - - logger.info('crawler finished at openid: {}'.format(openid)) + logger.info('crawler finished at openid: {}'.format(openid)) ''' @@ -346,12 +347,17 @@ def timer(request, _): return JsonResponse({'errcode': 0, 'errmsg': 'timer started'}, json_dumps_params={'ensure_ascii': False}) def looper(queryset): - index = 0 - while index < queryset.count(): + for user in queryset: sema.acquire() - user = queryset[index] Thread(target=worker, args=(user.openid,)).start() - index += 1 + + # index = 0 + # count = queryset.count() + # while index < count: + # sema.acquire() + # user = queryset[index] + # Thread(target=worker, args=(user.openid,)).start() + # index += 1 logger.info('timer finished') diff --git a/wxcloudrun/views.py b/wxcloudrun/views.py index b6396f4..981b3c0 100644 --- a/wxcloudrun/views.py +++ b/wxcloudrun/views.py @@ -232,5 +232,6 @@ def sessionExtend(openid): user.isLogin = False user.save() logger.error("session extend failed on openid: {} ".format(openid) + str(e.args)) + session = '' return session, user.sesskey -- Gitee From 4cc067be43acfec7d5b20eb8aa17b878af2fb1e8 Mon Sep 17 00:00:00 2001 From: "tanglinchuanz@163.com" Date: Sun, 13 Mar 2022 16:26:25 +0800 Subject: [PATCH 4/4] =?UTF-8?q?crontab=20log=20=E8=B7=AF=E5=BE=84=E4=BF=AE?= =?UTF-8?q?=E6=94=B9=EF=BC=8C=20db=E9=87=8D=E8=BF=9E=E4=B8=AD=E9=97=B4?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements.txt | 3 ++- wxcloudrun/settings.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 774e439..b33ba2f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ PyMySQL==1.0.2 requests==2.27.1 urllib3==1.26.8 pycryptodome==3.9.7 -django_crontab \ No newline at end of file +django_crontab +django_dbconn_retry \ No newline at end of file diff --git a/wxcloudrun/settings.py b/wxcloudrun/settings.py index a79665d..535a4ab 100644 --- a/wxcloudrun/settings.py +++ b/wxcloudrun/settings.py @@ -29,6 +29,7 @@ INSTALLED_APPS = [ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', + 'django_dbconn_retry', 'django_crontab', 'wxcloudrun', 'wxBackendsapi' @@ -192,7 +193,7 @@ LOGGING = { } CRONJOBS = [ - ('30 * * * *', 'wxBackendsapi.views.timer', ['',''], {}, '>> '+os.path.join(LOG_PATH, 'all-{}.log'.format(time.strftime('%Y-%m-%d')))) + ('30 * * * *', 'wxBackendsapi.views.timer', ['',''], {}, '>> /var/log/crontab.log') ] # Internationalization -- Gitee