diff --git a/requirements.txt b/requirements.txt index 774e439f941007a0f3fb6b1e19ae106a94583f5b..b33ba2fc1b661edce1ccf39659a75722a3771a08 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ PyMySQL==1.0.2 requests==2.27.1 urllib3==1.26.8 pycryptodome==3.9.7 -django_crontab \ No newline at end of file +django_crontab +django_dbconn_retry \ No newline at end of file diff --git a/wxBackendsapi/efficient_crawler.py b/wxBackendsapi/efficient_crawler.py index 5fc03798b6e393f441084ead720817dade9bb27b..acf56ac3f90312fc610558c8f8cafad4e2e42fa0 100644 --- a/wxBackendsapi/efficient_crawler.py +++ b/wxBackendsapi/efficient_crawler.py @@ -192,7 +192,7 @@ def CrawlerF(year, month, session, sesskey): days = week['days'] for day in days: if day['popovertitle'] != '': - ddlTime = str(day['year']) + ', ' + day['popovertitle'].replace(' events', '').replace(',', '') + # ddlTime = str(day['year']) + ', ' + day['popovertitle'].replace(' events', '').replace(',', '') # print(ddlTime) if day['events'] != []: """ @@ -224,7 +224,7 @@ def CrawlerF(year, month, session, sesskey): if event['normalisedeventtype'] == "course": course_info = event['course'] # shortname - shortname = course_info['shortname'][:199] + shortname = course_info['shortname'] shortname = cs.courseAbbr(shortname) data.append(shortname) # print(shortname) diff --git a/wxBackendsapi/models.py b/wxBackendsapi/models.py index acb3014903605bbaa97a2a2c0072c3efb7070998..81fdc2af571b9caf499c7c5012d8420d231c3a07 100644 --- a/wxBackendsapi/models.py +++ b/wxBackendsapi/models.py @@ -21,7 +21,7 @@ class DDLInfo(models.Model): openid = models.CharField("openid", max_length=100, default="") # username = models.CharField("Username", max_length=20, default="") # pwd = models.CharField("pwd", max_length=100, default="") - courseName = models.CharField("Course Name", max_length=200) + courseName = models.CharField("Course Name", max_length=100) DDL_Name = models.CharField("DDL Name", max_length=200) DDL_Time = models.CharField("DDL Time", max_length=100) # courseState = models.BooleanField("Course State", default=True) diff --git a/wxBackendsapi/views.py b/wxBackendsapi/views.py index f3ef325f3b9cfc7c9a34645c9e10137c61665283..f4acb294a4073f2fa017b184d268d1d235312f68 100644 --- a/wxBackendsapi/views.py +++ b/wxBackendsapi/views.py @@ -37,66 +37,68 @@ import wxBackendsapi.courseTimetableCrawler as wcourse def saveDDLInfo(openid, isLogin): session, sesskey = vi.sessionExtend(openid) - # get current month - today = datetime.datetime.today() - year = today.year - month = today.month - - queryset = models.DDLInfo.objects.filter(openid=openid) - - dict1 = we.CrawlerF(year, month, session, sesskey) - dict2 = we.CrawlerF(year+(month+1)//13, month%12+1, session, sesskey) - - data = {**dict1, **dict2} - - for ddlg in queryset: - if ddlg.eventID in data: - if ddlg.isShow == True: - # update ddl - ddl = data[ddlg.eventID] - if ddlg.DDL_Name != ddl[2]: - toTime = datetime.datetime.fromtimestamp(float(ddl[3])+8*3600).strftime('%m-%d %H:%M') - ddlg.DDL_Name = ddl[2] - ddlg.assState = True - if isLogin == False: - vi.notificationSend(os.environ.get('NOTIF_TEMPID_RELE'), openid, 'pages/list/list', ddl[1], ddl[2], toTime) - - if ddlg.DDL_Time != str(ddl[3]): - fromTime = datetime.datetime.fromtimestamp(float(ddlg.DDL_Time)+8*3600).strftime('%m-%d %H:%M') - toTime = datetime.datetime.fromtimestamp(float(ddl[3])+8*3600).strftime('%m-%d %H:%M') - ddlg.DDL_Time = ddl[3] - ddlg.assState = True - if isLogin == False: - vi.notificationSend(os.environ.get('NOTIF_TEMPID_RELE'), openid, 'pages/list/list', '时间更新'+ddl[1], ddl[2], fromTime+'~'+toTime) - - if str(ddlg.isSubmit) != str(ddl[4]): - ddlg.isSubmit = ddl[4] - - # ddl reminder - if str(ddl[4]) == 'False': - now = datetime.datetime.now().timestamp() - to = float(ddl[3]) - 3600*2 # 提前2小时 - if now + 3600 + 600 >= to and now <= to: # 检测范围为1h10m + if session != '': + # get current month + today = datetime.datetime.today() + year = today.year + month = today.month + + queryset = models.DDLInfo.objects.filter(openid=openid) + + dict1 = we.CrawlerF(year, month, session, sesskey) + dict2 = we.CrawlerF(year+(month+1)//13, month%12+1, session, sesskey) + + data = {**dict1, **dict2} + logger.info('dict: '+str(data)) + + for ddlg in queryset: + if ddlg.eventID in data: + if ddlg.isShow == True: + # update ddl + ddl = data[ddlg.eventID] + if ddlg.DDL_Name != ddl[2]: + toTime = datetime.datetime.fromtimestamp(float(ddl[3])+8*3600).strftime('%m-%d %H:%M') + ddlg.DDL_Name = ddl[2] ddlg.assState = True + if isLogin == False: + vi.notificationSend(os.environ.get('NOTIF_TEMPID_RELE'), openid, 'pages/list/list', ddl[1], ddl[2], toTime) + + if ddlg.DDL_Time != str(ddl[3]): + fromTime = datetime.datetime.fromtimestamp(float(ddlg.DDL_Time)+8*3600).strftime('%m-%d %H:%M') toTime = datetime.datetime.fromtimestamp(float(ddl[3])+8*3600).strftime('%m-%d %H:%M') + ddlg.DDL_Time = ddl[3] + ddlg.assState = True if isLogin == False: - vi.notificationSend(os.environ.get('NOTIF_TEMPID_URGE'), openid, 'pages/list/list', ddl[1], ddl[2], toTime) - + vi.notificationSend(os.environ.get('NOTIF_TEMPID_RELE'), openid, 'pages/list/list', '时间更新'+ddl[1], ddl[2], fromTime+'~'+toTime) + + if str(ddlg.isSubmit) != str(ddl[4]): + ddlg.isSubmit = ddl[4] + + # ddl reminder + if str(ddl[4]) == 'False': + now = datetime.datetime.now().timestamp() + to = float(ddl[3]) - 3600*2 # 提前2小时 + if now + 3600 + 600 >= to and now <= to: # 检测范围为1h10m + ddlg.assState = True + toTime = datetime.datetime.fromtimestamp(float(ddl[3])+8*3600).strftime('%m-%d %H:%M') + if isLogin == False: + vi.notificationSend(os.environ.get('NOTIF_TEMPID_URGE'), openid, 'pages/list/list', ddl[1], ddl[2], toTime) + + ddlg.save() + + del data[ddlg.eventID] + else: + # set isShow False + ddlg.isShow = False ddlg.save() - del data[ddlg.eventID] - else: - # set isShow False - ddlg.isShow = False - ddlg.save() + for ddl in data.values(): + # insert ddl + DDLInfo(eventID=ddl[0],openid=openid,courseName=ddl[1],DDL_Name=ddl[2],DDL_Time=ddl[3],isSubmit=ddl[4],moduleName=ddl[5]).save() + if isLogin == False: + vi.notificationSend(os.environ.get('NOTIF_TEMPID_RELE'), openid, 'pages/list/list', ddl[1], ddl[2], datetime.datetime.fromtimestamp(float(ddl[3])+8*3600).strftime('%m-%d %H:%M')) - for ddl in data.values(): - # insert ddl - DDLInfo(eventID=ddl[0],openid=openid,courseName=ddl[1],DDL_Name=ddl[2],DDL_Time=ddl[3],isSubmit=ddl[4],moduleName=ddl[5]).save() - if isLogin == False: - vi.notificationSend(os.environ.get('NOTIF_TEMPID_RELE'), openid, 'pages/list/list', ddl[1], ddl[2], datetime.datetime.fromtimestamp(float(ddl[3])+8*3600).strftime('%m-%d %H:%M')) - - logger.info('crawler finished at openid: {}'.format(openid)) + logger.info('crawler finished at openid: {}'.format(openid)) ''' @@ -113,8 +115,8 @@ def handCrawler(request): try: saveDDLInfo(rsp['openid'], True) except BaseException as e: - logger.error('crawler failed at openid: {} '.format(rsp['openid'])+str(e)) - return JsonResponse({'errcode': -6, 'errmsg': str(e)}, json_dumps_params={'ensure_ascii': False}) + logger.error('crawler failed at openid: {} '.format(rsp['openid'])+str(e.args)) + return JsonResponse({'errcode': -6, 'errmsg': str(e.args)}, json_dumps_params={'ensure_ascii': False}) return HttpResponse(json.dumps({"status": 200, "msg": "OK!", "IsUpdate":True}, ensure_ascii=False)) @@ -345,12 +347,17 @@ def timer(request, _): return JsonResponse({'errcode': 0, 'errmsg': 'timer started'}, json_dumps_params={'ensure_ascii': False}) def looper(queryset): - index = 0 - while index < queryset.count(): + for user in queryset: sema.acquire() - user = queryset[index] Thread(target=worker, args=(user.openid,)).start() - index += 1 + + # index = 0 + # count = queryset.count() + # while index < count: + # sema.acquire() + # user = queryset[index] + # Thread(target=worker, args=(user.openid,)).start() + # index += 1 logger.info('timer finished') @@ -359,6 +366,6 @@ def worker(openid): try: saveDDLInfo(openid, False) except BaseException as e: - logger.error('crawler failed on openid {} '.format(openid) + str(e)) + logger.error('crawler failed on openid {} '.format(openid) + str(e.args)) finally: sema.release() \ No newline at end of file diff --git a/wxcloudrun/settings.py b/wxcloudrun/settings.py index db6e56acb10b7da5435e61ed5464441526462d22..535a4ab944d29ac52e7fe3c3fd8cb13651b1b757 100644 --- a/wxcloudrun/settings.py +++ b/wxcloudrun/settings.py @@ -29,6 +29,7 @@ INSTALLED_APPS = [ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', + 'django_dbconn_retry', 'django_crontab', 'wxcloudrun', 'wxBackendsapi' diff --git a/wxcloudrun/views.py b/wxcloudrun/views.py index 041408acc8d72fba6d5ef8ded658f04780410f4d..981b3c01f3a8a3a44a143ab710158d6906f8b770 100644 --- a/wxcloudrun/views.py +++ b/wxcloudrun/views.py @@ -24,7 +24,7 @@ def tokenValidate(request): if user.isLogin == False: raise Exception("session expired", user.openid) except BaseException as e: - ret = {'errcode': -1, 'errorMsg': str(e)} + ret = {'errcode': -1, 'errorMsg': str(e.args)} else: ret = {'openid': token['openid']} else: @@ -103,7 +103,7 @@ def uicAuth(username, password): try: rsp = session.get(loginUrl, verify=False) except BaseException as e: - logger.error('request result: '+str(e)) + logger.error('request result: '+str(e.args)) else: loginToken = BeautifulSoup(rsp.text, 'html.parser').find("input",{'name': 'logintoken'})['value'] @@ -121,7 +121,7 @@ def uicAuth(username, password): try: rsp = session.post(loginUrl, postdata, headers=header, verify=False) except BaseException as e: - logger.error('request result: '+str(e)) + logger.error('request result: '+str(e.args)) else: sesskey = BeautifulSoup(rsp.text, 'html.parser').find("input",{'name': 'sesskey'})['value'] session = requests.utils.dict_from_cookiejar(session.cookies)['MoodleSession'] @@ -144,7 +144,7 @@ def weixinAuth(js_code): try: rsp = requests.get(url, params=payload, verify=False) except BaseException as e: - logger.error('request result: '+str(e)) + logger.error('request result: '+str(e.args)) else: logger.info('weixinAuth result: {}'.format(rsp.content.decode('utf-8'))) finally: @@ -231,6 +231,7 @@ def sessionExtend(openid): user = models.UserInfo.objects.get(openid=openid) user.isLogin = False user.save() - logger.error("session extend failed on openid: {} ".format(openid) + str(e)) + logger.error("session extend failed on openid: {} ".format(openid) + str(e.args)) + session = '' return session, user.sesskey