From b41812255d7d8b4ed8c6a82b6197576b2eba66dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8C=97=E8=8C=97=E6=98=AF=E5=8F=AA=E5=81=87=E9=B8=BD?= =?UTF-8?q?=E5=AD=90?= <471167673@qq.com> Date: Sun, 14 Mar 2021 16:39:10 +0800 Subject: [PATCH 1/6] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20week11?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../week11/.keep" | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 "\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/.keep" diff --git "a/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/.keep" "b/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/.keep" new file mode 100644 index 00000000..e69de29b -- Gitee From f26589e0e0ba24477d69d71dfdb0b8391de2646e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8C=97=E8=8C=97=E6=98=AF=E5=8F=AA=E5=81=87=E9=B8=BD?= =?UTF-8?q?=E5=AD=90?= <471167673@qq.com> Date: Sun, 14 Mar 2021 16:39:27 +0800 Subject: [PATCH 2/6] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20lesson1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../week11/lesson1/.keep" | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 "\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson1/.keep" diff --git "a/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson1/.keep" "b/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson1/.keep" new file mode 100644 index 00000000..e69de29b -- Gitee From 1458934e69b994623c2ecb56736ba7b12dd2b866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8C=97=E8=8C=97=E6=98=AF=E5=8F=AA=E5=81=87=E9=B8=BD?= =?UTF-8?q?=E5=AD=90?= <471167673@qq.com> Date: Sun, 14 Mar 2021 16:40:53 +0800 Subject: [PATCH 3/6] =?UTF-8?q?=E7=AC=AC=E5=8D=81=E4=B8=80=E5=91=A8-?= =?UTF-8?q?=E7=AC=AC=E4=B8=80=E8=8A=82-=E4=BD=9C=E4=B8=9A=E6=8F=90?= =?UTF-8?q?=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../week11/lesson1/middlewares.py" | 143 ++++++++++++++++++ .../week11/lesson1/settings.py" | 111 ++++++++++++++ 2 files changed, 254 insertions(+) create mode 100644 "\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson1/middlewares.py" create mode 100644 "\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson1/settings.py" diff --git "a/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson1/middlewares.py" "b/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson1/middlewares.py" new file mode 100644 index 00000000..a4243bf4 --- /dev/null +++ "b/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson1/middlewares.py" @@ -0,0 +1,143 @@ +# Define here the models for your spider middleware +# +# See documentation in: +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html + +from scrapy import signals +from scrapy.downloadermiddlewares.retry import RetryMiddleware +from scrapy.utils.response import response_status_message +from scrapy.dupefilters import RFPDupeFilter +from itemadapter import is_item, ItemAdapter +from w3lib.url import canonicalize_url +from scrapy.utils.python import to_bytes, to_unicode + +import hashlib +import weakref + + + +# useful for handling different item types with a single interface + + +_fingerprint_cache = weakref.WeakKeyDictionary() + + +class JdCrawlerScrapySpiderMiddleware: + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the spider middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_spider_input(self, response, spider): + # Called for each response that goes through the spider + # middleware and into the spider. + + # Should return None or raise an exception. + return None + + def process_spider_output(self, response, result, spider): + # Called with the results returned from the Spider, after + # it has processed the response. + + # Must return an iterable of Request, or item objects. + for i in result: + yield i + + def process_spider_exception(self, response, exception, spider): + # Called when a spider or process_spider_input() method + # (from other spider middleware) raises an exception. + + # Should return either None or an iterable of Request or item objects. + pass + + def process_start_requests(self, start_requests, spider): + # Called with the start requests of the spider, and works + # similarly to the process_spider_output() method, except + # that it doesn’t have a response associated. + + # Must return only requests (not items). + for r in start_requests: + yield r + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) + + +class JdCrawlerScrapyDownloaderMiddleware: + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the downloader middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_request(self, request, spider): + # Called for each request that goes through the downloader + # middleware. + + # Must either: + # - return None: continue processing this request + # - or return a Response object + # - or return a Request object + # - or raise IgnoreRequest: process_exception() methods of + # installed downloader middleware will be called + return None + + def process_response(self, request, response, spider): + # Called with the response returned from the downloader. + + # Must either; + # - return a Response object + # - return a Request object + # - or raise IgnoreRequest + return response + + def process_exception(self, request, exception, spider): + # Called when a download handler or a process_request() + # (from other downloader middleware) raises an exception. + + # Must either: + # - return None: continue processing this exception + # - return a Response object: stops process_exception() chain + # - return a Request object: stops process_exception() chain + pass + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) + +class UAMiddleware: + def process_request(self, request, spider): + request.headers["user-agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36 Edg/88.0.705.81" + +class MyRetryMiddleware(RetryMiddleware): + def process_response(self, request, response, spider): + if request.meta.get('dont_retry', False): + return response + if "验证码" in response.text: + reason = response_status_message(response.status) + return self._retry(request, reason, spider) or response + return response + + +class MyRFPDupeFilter(RFPDupeFilter): + def request_fingerprint(self, request, include_headers=None, keep_fragments=False): + cache = _fingerprint_cache.setdefault(request, {}) + cache_key = (include_headers, keep_fragments) + if cache_key not in cache: + fp = hashlib.sha1() + fp.update(to_bytes(request.method)) + fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))) + fp.update(request.body or b'') + fp.update(request.meta.get("batch_no", "").encode("utf-8")) + cache[cache_key] = fp.hexdigest() + return cache[cache_key] diff --git "a/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson1/settings.py" "b/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson1/settings.py" new file mode 100644 index 00000000..cad027e2 --- /dev/null +++ "b/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson1/settings.py" @@ -0,0 +1,111 @@ +# Scrapy settings for jd_crawler_scrapy project +# +# For simplicity, this file contains only settings considered important or +# commonly used. You can find more settings consulting the documentation: +# +# https://docs.scrapy.org/en/latest/topics/settings.html +# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html + +BOT_NAME = 'jd_crawler_scrapy' + +SPIDER_MODULES = ['jd_crawler_scrapy.spiders'] +NEWSPIDER_MODULE = 'jd_crawler_scrapy.spiders' + + +# Crawl responsibly by identifying yourself (and your website) on the user-agent +#USER_AGENT = 'jd_crawler_scrapy (+http://www.yourdomain.com)' + +# Obey robots.txt rules +ROBOTSTXT_OBEY = False + +REDIRECT_ENABLE = False + +RETRY_ENABLE = False +RETRY_HTTP_CODES = [500, 502, 503, 504, 408, 429] + +# Configure maximum concurrent requests performed by Scrapy (default: 16) +CONCURRENT_REQUESTS = 1 + + +# Configure a delay for requests for the same website (default: 0) +# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay +# See also autothrottle settings and docs +DOWNLOAD_DELAY = 3 +# The download delay setting will honor only one of: +#CONCURRENT_REQUESTS_PER_DOMAIN = 16 +#CONCURRENT_REQUESTS_PER_IP = 16 + +# Disable cookies (enabled by default) +#COOKIES_ENABLED = False + +# Disable Telnet Console (enabled by default) +#TELNETCONSOLE_ENABLED = False + +# Override the default request headers: +#DEFAULT_REQUEST_HEADERS = { +# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', +# 'Accept-Language': 'en', +#} + +# Enable or disable spider middlewares +# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html +#SPIDER_MIDDLEWARES = { +# 'jd_crawler_scrapy.middlewares.JdCrawlerScrapySpiderMiddleware': 543, +#} + +# Enable or disable downloader middlewares +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html +DOWNLOADER_MIDDLEWARES = { + # 'jd_crawler_scrapy.middlewares.JdCrawlerScrapyDownloaderMiddleware': 543, + 'jd_crawler_scrapy.middlewares.UAMiddleware': 100, + 'jd_crawler_scrapy.middlewares.MyRetryMiddleware': 200, +} + +# Enable or disable extensions +# See https://docs.scrapy.org/en/latest/topics/extensions.html +#EXTENSIONS = { +# 'scrapy.extensions.telnet.TelnetConsole': None, +#} + +# Configure item pipelines +# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html +ITEM_PIPELINES = { + 'jd_crawler_scrapy.pipelines.JdCrawlerScrapyPipeline': 300, +} + +# Enable and configure the AutoThrottle extension (disabled by default) +# See https://docs.scrapy.org/en/latest/topics/autothrottle.html +#AUTOTHROTTLE_ENABLED = True +# The initial download delay +#AUTOTHROTTLE_START_DELAY = 5 +# The maximum download delay to be set in case of high latencies +#AUTOTHROTTLE_MAX_DELAY = 60 +# The average number of requests Scrapy should be sending in parallel to +# each remote server +#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 +# Enable showing throttling stats for every response received: +#AUTOTHROTTLE_DEBUG = False + +# Enable and configure HTTP caching (disabled by default) +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings +#HTTPCACHE_ENABLED = True +#HTTPCACHE_EXPIRATION_SECS = 0 +#HTTPCACHE_DIR = 'httpcache' +#HTTPCACHE_IGNORE_HTTP_CODES = [] +#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' + + +MYSQL_CONF = { + "host": "127.0.0.1", + "user": "root", + "password": "xmt66237029", + "db": "week9" +} + +# LOG +LOG_FILE = "D:/log/jd_search.log" +LOG_LEVEL = "DEBUG" + +# DUP +DUPEFILTER_CLASS = "jd_crawler_scrapy.middlewares.MyRFPDupeFilter" \ No newline at end of file -- Gitee From 35d96c679a73d606933230c75ff48c8fdb2cc5ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8C=97=E8=8C=97=E6=98=AF=E5=8F=AA=E5=81=87=E9=B8=BD?= =?UTF-8?q?=E5=AD=90?= <471167673@qq.com> Date: Sun, 14 Mar 2021 16:41:03 +0800 Subject: [PATCH 4/6] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20lesson2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../week11/lesson2/.keep" | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 "\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/.keep" diff --git "a/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/.keep" "b/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/.keep" new file mode 100644 index 00000000..e69de29b -- Gitee From da1d8008124e9245267d8d577d943100d2598f5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8C=97=E8=8C=97=E6=98=AF=E5=8F=AA=E5=81=87=E9=B8=BD?= =?UTF-8?q?=E5=AD=90?= <471167673@qq.com> Date: Sun, 14 Mar 2021 21:03:33 +0800 Subject: [PATCH 5/6] =?UTF-8?q?=E7=AC=AC=E5=8D=81=E4=B8=80=E5=91=A8-?= =?UTF-8?q?=E7=AC=AC=E4=BA=8C=E8=8A=82-=E4=BD=9C=E4=B8=9A=E6=8F=90?= =?UTF-8?q?=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../week11/lesson2/__init__.py" | 0 .../week11/lesson2/items.py" | 19 ++ .../week11/lesson2/middlewares.py" | 163 ++++++++++++++++++ .../week11/lesson2/pipelines.py" | 29 ++++ .../week11/lesson2/settings.py" | 113 ++++++++++++ .../week11/lesson2/spiders/__init__.py" | 4 + .../__pycache__/__init__.cpython-38.pyc" | Bin 0 -> 182 bytes .../__pycache__/jd_search.cpython-38.pyc" | Bin 0 -> 2439 bytes .../spiders/__pycache__/run.cpython-38.pyc" | Bin 0 -> 286 bytes .../week11/lesson2/spiders/jd_search.py" | 64 +++++++ .../week11/lesson2/spiders/run.py" | 4 + 11 files changed, 396 insertions(+) create mode 100644 "\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/__init__.py" create mode 100644 "\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/items.py" create mode 100644 "\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/middlewares.py" create mode 100644 "\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/pipelines.py" create mode 100644 "\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/settings.py" create mode 100644 "\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/spiders/__init__.py" create mode 100644 "\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/spiders/__pycache__/__init__.cpython-38.pyc" create mode 100644 "\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/spiders/__pycache__/jd_search.cpython-38.pyc" create mode 100644 "\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/spiders/__pycache__/run.cpython-38.pyc" create mode 100644 "\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/spiders/jd_search.py" create mode 100644 "\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/spiders/run.py" diff --git "a/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/__init__.py" "b/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/__init__.py" new file mode 100644 index 00000000..e69de29b diff --git "a/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/items.py" "b/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/items.py" new file mode 100644 index 00000000..1c96e07c --- /dev/null +++ "b/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/items.py" @@ -0,0 +1,19 @@ +# Define here the models for your scraped items +# +# See documentation in: +# https://docs.scrapy.org/en/latest/topics/items.html + +import scrapy + + +class JdCrawlerScrapyItem(scrapy.Item): + # define the fields for your item here like: + # name = scrapy.Field() + sku_id = scrapy.Field() + img = scrapy.Field() + price = scrapy.Field() + title = scrapy.Field() + shop = scrapy.Field() + icons = scrapy.Field() + sta_date = scrapy.Field() + keyword = scrapy.Field() diff --git "a/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/middlewares.py" "b/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/middlewares.py" new file mode 100644 index 00000000..50871150 --- /dev/null +++ "b/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/middlewares.py" @@ -0,0 +1,163 @@ +# Define here the models for your spider middleware +# +# See documentation in: +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html + +from scrapy import signals +from scrapy.downloadermiddlewares.retry import RetryMiddleware +from scrapy.utils.response import response_status_message +from scrapy.dupefilters import RFPDupeFilter +from itemadapter import is_item, ItemAdapter +from w3lib.url import canonicalize_url +from scrapy.utils.python import to_bytes, to_unicode + +import hashlib +import weakref + + + +# useful for handling different item types with a single interface + + +_fingerprint_cache = weakref.WeakKeyDictionary() + + +class JdCrawlerScrapySpiderMiddleware: + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the spider middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_spider_input(self, response, spider): + # Called for each response that goes through the spider + # middleware and into the spider. + + # Should return None or raise an exception. + return None + + def process_spider_output(self, response, result, spider): + # Called with the results returned from the Spider, after + # it has processed the response. + + # Must return an iterable of Request, or item objects. + for i in result: + yield i + + def process_spider_exception(self, response, exception, spider): + # Called when a spider or process_spider_input() method + # (from other spider middleware) raises an exception. + + # Should return either None or an iterable of Request or item objects. + pass + + def process_start_requests(self, start_requests, spider): + # Called with the start requests of the spider, and works + # similarly to the process_spider_output() method, except + # that it doesn’t have a response associated. + + # Must return only requests (not items). + for r in start_requests: + yield r + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) + + +class JdCrawlerScrapyDownloaderMiddleware: + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the downloader middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_request(self, request, spider): + # Called for each request that goes through the downloader + # middleware. + + # Must either: + # - return None: continue processing this request + # - or return a Response object + # - or return a Request object + # - or raise IgnoreRequest: process_exception() methods of + # installed downloader middleware will be called + return None + + def process_response(self, request, response, spider): + # Called with the response returned from the downloader. + + # Must either; + # - return a Response object + # - return a Request object + # - or raise IgnoreRequest + return response + + def process_exception(self, request, exception, spider): + # Called when a download handler or a process_request() + # (from other downloader middleware) raises an exception. + + # Must either: + # - return None: continue processing this exception + # - return a Response object: stops process_exception() chain + # - return a Request object: stops process_exception() chain + pass + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) + + + +class CookieMiddleware: + def process_request(self, request, spider): + # request.headers['cookie'] = "__jdu=16145053508501715621824; shshshfpa=04eb19ca-e4a4-e402-4ba1-661209b44b45-1614505356; shshshfpb=pfquKaGf2BF9BEAn5FVoKTQ%3D%3D; qrsc=3; unpl=V2_ZzNtbUcCRkUlCE8Ech9eAmIGGg1LUUURJwBDUCsRC1BiVBUKclRCFnUUR1ZnGVkUZwoZWEpcQBRFCEdkeBBVAWMDE1VGZxBFLV0CFSNGF1wjU00zQwBBQHcJFF0uSgwDYgcaDhFTQEJ2XBVQL0oMDDdRFAhyZ0AVRQhHZHseXABjAhZfRlZKEXQIQFR%2fHl8FbgAXbXJQcyVFCEZSfRtdNWYzE20AAx8cdAhEVXlUXAJnBhZcRlVHFHwMR1R9GVgCZAMbXkdnQiV2; __jdv=76161171|baidu-pinzhuan|t_288551095_baidupinzhuan|cpc|0f3d30c8dba7459bb52f2eb5eba8ac7d_0_4d4aa18a8736449a8775c945a9fd4f6f|1615718959550; areaId=12; ipLoc-djd=12-988-40034-0; PCSYCityID=CN_0_0_0; __jda=122270672.16145053508501715621824.1614505351.1614530307.1615718960.4; __jdb=122270672.2.16145053508501715621824|4.1615718960; __jdc=122270672; shshshfp=c907bac07f57d2d9bd6b95e4bbd98546; shshshsID=dd3ddb3830270a6da29617dd323e8665_2_1615718964705; rkv=1.0; 3AB9D23F7A4B3C9B=UZVSMFBYANBA7S4P2CFROVSTUANCYW6XIJDGKUWUAK76VESYQC6EAFH3EXFMOYCWPYX3DK3TKKQ3XROBBURIVBFCMM" + # cookie_pool = [] + # request.headers['cookie'] = cookie_pool.pop() + pass + + def process_response(self, request, response, spider): + # if "身份过期" in response.text: + raise Exception("当前cookie身份已过期") + + def process_exception(self, request, exception, spider): + if isinstance(exception, IndexError): + retry_times = request.meta.get('retry_times', 1) + request.meta['retry_times'] = retry_times - 1 + return request + + +class UAMiddleware: + def process_request(self, request, spider): + request.headers["user-agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36 Edg/88.0.705.81" + +class MyRetryMiddleware(RetryMiddleware): + def process_response(self, request, response, spider): + if request.meta.get('dont_retry', False): + return response + if "验证码" in response.text: + reason = response_status_message(response.status) + return self._retry(request, reason, spider) or response + return response + + +class MyRFPDupeFilter(RFPDupeFilter): + def request_fingerprint(self, request, include_headers=None, keep_fragments=False): + cache = _fingerprint_cache.setdefault(request, {}) + cache_key = (include_headers, keep_fragments) + if cache_key not in cache: + fp = hashlib.sha1() + fp.update(to_bytes(request.method)) + fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))) + fp.update(request.body or b'') + fp.update(request.meta.get("batch_no", "").encode("utf-8")) + cache[cache_key] = fp.hexdigest() + return cache[cache_key] diff --git "a/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/pipelines.py" "b/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/pipelines.py" new file mode 100644 index 00000000..3a8e610b --- /dev/null +++ "b/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/pipelines.py" @@ -0,0 +1,29 @@ +# Define your item pipelines here +# +# Don't forget to add your pipeline to the ITEM_PIPELINES setting +# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html + + +# useful for handling different item types with a single interface +from itemadapter import ItemAdapter +from jd_crawler_scrapy.items import JdCrawlerScrapyItem +import pymysql + + +class JdCrawlerScrapyPipeline: + def __init__(self): + self.mysql_con = None + + def process_item(self, item, spider): + if not self.mysql_con: + self.mysql_con = pymysql.connect(**spider.settings["MYSQL_CONF"]) + + if isinstance(item, JdCrawlerScrapyItem): + cursor = self.mysql_con.cursor() + SQL = """insert into jd_search(sku_id, img, price, title, shop, icons, sta_date, keyword) values ('{}', '{}', '{}' , '{}', '{}', '{}', '{}', '{}')""".format( + item['sku_id'], item['img'], item['price'], item['title'], item['shop'], item['icons'], item['sta_date'], item['keyword'] + ) + cursor.execute(SQL) + self.mysql_con.commit() + cursor.close() + return item diff --git "a/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/settings.py" "b/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/settings.py" new file mode 100644 index 00000000..f7ee7fda --- /dev/null +++ "b/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/settings.py" @@ -0,0 +1,113 @@ +# Scrapy settings for jd_crawler_scrapy project +# +# For simplicity, this file contains only settings considered important or +# commonly used. You can find more settings consulting the documentation: +# +# https://docs.scrapy.org/en/latest/topics/settings.html +# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html + +BOT_NAME = 'jd_crawler_scrapy' + +SPIDER_MODULES = ['jd_crawler_scrapy.spiders'] +NEWSPIDER_MODULE = 'jd_crawler_scrapy.spiders' + + +# Crawl responsibly by identifying yourself (and your website) on the user-agent +#USER_AGENT = 'jd_crawler_scrapy (+http://www.yourdomain.com)' + +# Obey robots.txt rules +ROBOTSTXT_OBEY = False + +REDIRECT_ENABLE = False + +RETRY_ENABLE = False +RETRY_HTTP_CODES = [500, 502, 503, 504, 408, 429] + +# Configure maximum concurrent requests performed by Scrapy (default: 16) +CONCURRENT_REQUESTS = 1 + + +# Configure a delay for requests for the same website (default: 0) +# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay +# See also autothrottle settings and docs +DOWNLOAD_DELAY = 3 +# The download delay setting will honor only one of: +#CONCURRENT_REQUESTS_PER_DOMAIN = 16 +#CONCURRENT_REQUESTS_PER_IP = 16 + +# Disable cookies (enabled by default) +#COOKIES_ENABLED = False + +# Disable Telnet Console (enabled by default) +#TELNETCONSOLE_ENABLED = False + +# Override the default request headers: +#DEFAULT_REQUEST_HEADERS = { +# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', +# 'Accept-Language': 'en', +#} + +# Enable or disable spider middlewares +# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html +#SPIDER_MIDDLEWARES = { +# 'jd_crawler_scrapy.middlewares.JdCrawlerScrapySpiderMiddleware': 543, +#} + +# Enable or disable downloader middlewares +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html +DOWNLOADER_MIDDLEWARES = { + # 'jd_crawler_scrapy.middlewares.JdCrawlerScrapyDownloaderMiddleware': 543, + 'jd_crawler_scrapy.middlewares.UAMiddleware': 100, + 'jd_crawler_scrapy.middlewares.MyRetryMiddleware': 200, + 'jd_crawler_scrapy.middlewares.CookieMiddleware': 150, + +} + +# Enable or disable extensions +# See https://docs.scrapy.org/en/latest/topics/extensions.html +#EXTENSIONS = { +# 'scrapy.extensions.telnet.TelnetConsole': None, +#} + +# Configure item pipelines +# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html +ITEM_PIPELINES = { + 'jd_crawler_scrapy.pipelines.JdCrawlerScrapyPipeline': 300, +} + +# Enable and configure the AutoThrottle extension (disabled by default) +# See https://docs.scrapy.org/en/latest/topics/autothrottle.html +#AUTOTHROTTLE_ENABLED = True +# The initial download delay +#AUTOTHROTTLE_START_DELAY = 5 +# The maximum download delay to be set in case of high latencies +#AUTOTHROTTLE_MAX_DELAY = 60 +# The average number of requests Scrapy should be sending in parallel to +# each remote server +#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 +# Enable showing throttling stats for every response received: +#AUTOTHROTTLE_DEBUG = False + +# Enable and configure HTTP caching (disabled by default) +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings +#HTTPCACHE_ENABLED = True +#HTTPCACHE_EXPIRATION_SECS = 0 +#HTTPCACHE_DIR = 'httpcache' +#HTTPCACHE_IGNORE_HTTP_CODES = [] +#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' + + +MYSQL_CONF = { + "host": "127.0.0.1", + "user": "root", + "password": "xmt66237029", + "db": "week9" +} + +# LOG +# LOG_FILE = "D:/log/jd_search.log" +LOG_LEVEL = "DEBUG" + +# DUP +DUPEFILTER_CLASS = "jd_crawler_scrapy.middlewares.MyRFPDupeFilter" \ No newline at end of file diff --git "a/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/spiders/__init__.py" "b/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/spiders/__init__.py" new file mode 100644 index 00000000..ebd689ac --- /dev/null +++ "b/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/spiders/__init__.py" @@ -0,0 +1,4 @@ +# This package will contain the spiders of your Scrapy project +# +# Please refer to the documentation for information on how to create and manage +# your spiders. diff --git "a/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/spiders/__pycache__/__init__.cpython-38.pyc" "b/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/spiders/__pycache__/__init__.cpython-38.pyc" new file mode 100644 index 0000000000000000000000000000000000000000..a5aa3f6872be2c5db2d5616e3974bf013526f9b0 GIT binary patch literal 182 zcmWIL<>g`kf*-405*7mK#~=Z1E)SmIq&dzSqHf<9E5~QVRScy#lfrL;}5G^1o5mf|Qm?j#Av6IYZW@lo1 zX|kF{h*aWKsYo0uwTR?`gpfGE0g2B$KfzZZG&ipF#Iq+M3GJ->zHC3g_x#@TzCAGD zA^1-H^TvBW))4w#GY3Bc%u6uDn*b=HI6;1iv5yUJ`4(U+u~Xs`tmo{ck~+R)Xp&S@ z*LN{`4bcj9t|RI&)UIvVzDMyKYF2kZFPd#)HSjVEyE0zvCUaRg2f^`+^gIu*C5+ES zJj~Z$mn;Qw{d|%MHkZeg@knnz_*uZb1VcOlfFW%nn1_9vTGYOdd_oDW00vWN6|h5H zFrud1i*$~KJX-c^Z5jweMi_MLeL4<0Qu;*U1EBZ@AsCle2}NK|eYQ!u4xF*` z)8_W(CnfpWH-Fu}WytMYzx;V)%aC_J`fmHq4?Eh>ojQQt__CC_cwuU)bYZehC!;K# zDt&u#g{@!9IGyP^XY+80&Gg*ycU$9WYn(PMRo$Icu8?6sL&;uM_G_=a)pQg|nG98( zW*r$U#))KH*Jq?OP!yAjOS)gT0f%Yc{0 ziw2x19s+={iwE&hOaNNuM=-hPxxJCSY??g0Gt%j~_h~iy&P+yKhbxq*z#AANi3`XY ziZ`sjRal#lx?8w!7xo2o6EFCEaufH?6r@e4J&O93g1`y_D;z^rfT|h_@~!V0stQ!i zP%cnj;qYGJ(Ap>h9o{JFShNnitNdnBHPTAkX5(m{n+?2$jebuyHc+1w-~~N8jS8@w9s>;4 z(jma$6Fm+X{GbiM9^eyzaeIKC97VvL+~tN1Hw+xn^>hWTo+d~Ru6Q>wFEHxYG3pIM zgzN1iI>uMfbI_^!^$+Xspm$;IJ1|V`jm#>NtffisnQn45NvzbeC}fI$W6sw=_6Kq+rJ ziG_qtlb?H1Yt-G+K^E#Y|Eq+96+StgWY99iIUx4-DI7zuSm_2aRTUE*D0}c#+1iPU z@sPs{g-jo49yN|A_Hn5bMy1=K<(O(!ieOK`Zh0f7US8zccu(<-m&l-yk#`&lqWAC~sV@K(a(W6gGtz^c1HeEr9dKmYmtw|9U2 zWc$t+O~Unys%(g(s(cV8UB>k#H?1;2)x|JQy8J#jM_|ni456R1vS0Rw_l-W-=V#%~ zaDzbSeh?@x2-1vp6HPaQV6_`2yE8;%{s^cuLFD>~1Nmq&Q%9Cp0O*7f`r&XfvD`uT zaU*Tkxz2e$rNIdRzWu&<)})d@a^EvCsV}(95_PP6IVRa&o(K~L<&^(NjP?f^qulAz SopZWG%@a~yX9F(2VgC!4cYf6X literal 0 HcmV?d00001 diff --git "a/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/spiders/__pycache__/run.cpython-38.pyc" "b/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/spiders/__pycache__/run.cpython-38.pyc" new file mode 100644 index 0000000000000000000000000000000000000000..936fe43bf52dd38455fa565106a45b9a68fbd82a GIT binary patch literal 286 zcmZvVziPuU5XL3jsY6KC&Y_!^K0qm@L*@`D?GRKFqdTxu$r2}<#C@7%>?>*QROl0Q z>Lp$}^n?43@BZ+6p3NqL_4>Wo0{72z_^Xt|BUiX#0|C+qE7wvANRG)#tW%W74Aawj zl0Jzd`{FxM$e3#jrf%^4I%q0=KWE>Y`3^P#OVZV{7{#k$MG3#eQShd2`OjHfHV|_> zpzd1y5R~)&Vm~j7JLP*_m1Naw=qqwN)NN3ev$%KkrkHVoA#Dc0Ka=2VKnjYw=GOOd Sd~Y3e1|KA&LdG^t#_|{2SW?sg literal 0 HcmV?d00001 diff --git "a/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/spiders/jd_search.py" "b/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/spiders/jd_search.py" new file mode 100644 index 00000000..33bcf654 --- /dev/null +++ "b/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/spiders/jd_search.py" @@ -0,0 +1,64 @@ +import scrapy +import json +import time +from bs4 import BeautifulSoup +from jd_crawler_scrapy.items import JdCrawlerScrapyItem +from scrapy.exceptions import CloseSpider + + +class JdSearch(scrapy.Spider): + name = "jd_search" + + def start_requests(self): + for keyword in ["鼠标", "键盘", "显卡", "耳机"]: + for page_num in range(1, 11): + url = f"https://search.jd.com/Search?keyword={keyword}&page={page_num}" + yield scrapy.FormRequest( + meta={"keyword": keyword, "sta_date": time.strftime("%Y-%m-%d")}, + dont_filter=False, + url=url, + method='GET', + callback=self.parse_search, + errback=self.process_error + ) + break + + def parse_search(self, response): + print(response) + soup = BeautifulSoup(response.text, "lxml") + item_array = soup.select("ul[class = 'gl-warp clearfix'] li[class='gl-item']") + + for item in item_array: + try: + sku_id = item.attrs["data-sku"] + img = item.select("img[data-img='1']") + price = item.select("div[class='p-price']") + title = item.select("div[class='p-name p-name-type-2']") + shop = item.select("div[class='p-shop']") + icons = item.select("div[class='p-icons']") + + img = img[0].attrs['data-lazy-img'] if img else "" + price = price[0].strong.i.text if price else "" + title = title[0].text.strip() if title else "" + shop = shop[0].span.a.attrs['title'] if shop[0].text.strip() else "" + icons = json.dumps([tag_ele.text for tag_ele in icons[0].select("i")]) if icons else "" + + item = JdCrawlerScrapyItem() + item["sku_id"] = sku_id + item["price"] = price + item["shop"] = shop + item["img"] = img + item["title"] = title + item["icons"] = icons + item["sta_date"] = response.meta["sta_date"] + item["keyword"] = response.meta["keyword"] + yield item + + except Exception as e: + print(e.args) + + + def process_error(self, failure): + print(failure) + if "身份已过期" in str(failure.value): + raise CloseSpider diff --git "a/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/spiders/run.py" "b/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/spiders/run.py" new file mode 100644 index 00000000..81ab4335 --- /dev/null +++ "b/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/lesson2/spiders/run.py" @@ -0,0 +1,4 @@ +from scrapy import cmdline + +command = "scrapy crawl jd_search".split() +cmdline.execute(command) -- Gitee From d42bf6c697d23721f5feebead5fe537446292b25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8C=97=E8=8C=97=E6=98=AF=E5=8F=AA=E5=81=87=E9=B8=BD?= =?UTF-8?q?=E5=AD=90?= <471167673@qq.com> Date: Sun, 14 Mar 2021 23:19:16 +0800 Subject: [PATCH 6/6] =?UTF-8?q?=E7=AC=AC=E5=8D=81=E4=B8=80=E5=91=A8-?= =?UTF-8?q?=E7=AC=AC=E4=B8=89=E8=8A=82-=E4=BD=9C=E4=B8=9A=E6=8F=90?= =?UTF-8?q?=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...0-\347\254\254\344\270\211\350\212\202.md" | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 "\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/\347\254\254\345\215\201\344\270\200\345\221\250-\347\254\254\344\270\211\350\212\202.md" diff --git "a/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/\347\254\254\345\215\201\344\270\200\345\221\250-\347\254\254\344\270\211\350\212\202.md" "b/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/\347\254\254\345\215\201\344\270\200\345\221\250-\347\254\254\344\270\211\350\212\202.md" new file mode 100644 index 00000000..419e9a03 --- /dev/null +++ "b/\347\254\254\344\272\214\346\234\237\350\256\255\347\273\203\350\220\245/3\347\217\255/3\347\217\255_\350\214\227\350\214\227\346\230\257\345\217\252\345\201\207\351\270\275\345\255\220/week11/\347\254\254\345\215\201\344\270\200\345\221\250-\347\254\254\344\270\211\350\212\202.md" @@ -0,0 +1,99 @@ +# 第十一周-第三节 + +``` +C:\Users\47116>redis-server +[29224] 14 Mar 21:44:54.017 # Warning: no config file specified, using the default config. In order to specify a config file use redis-server /path/to/redis.conf +[29224] 14 Mar 21:44:54.021 # Creating Server TCP listening socket *:6379: bind: No such file or directory + +C:\Users\47116>redis-cli +127.0.0.1:6379> select 2 +OK +127.0.0.1:6379[2]> keys * +(empty list or set) +127.0.0.1:6379[2]> set 'a' 1 +OK +127.0.0.1:6379[2]> get 'a' +"1" +127.0.0.1:6379[2]> set 'a' 2 +OK +127.0.0.1:6379[2]> del 'a' +(integer) 1 +127.0.0.1:6379[2]> keys * +(empty list or set) +127.0.0.1:6379[2]> set 'a' 1 +OK +127.0.0.1:6379[2]> hset 'hash_test' 'a' 1 +(integer) 1 +127.0.0.1:6379[2]> hset 'hash_test' 'b' 2 +(integer) 1 +127.0.0.1:6379[2]> hget 'hash_test' 'a' +"1" +127.0.0.1:6379[2]> hdel 'hash_test' 'a' +(integer) 1 +127.0.0.1:6379[2]> lpush 'list_test' 1 +(integer) 1 +127.0.0.1:6379[2]> lpush 'list_test' 2 +(integer) 2 +127.0.0.1:6379[2]> lpush 'list_test' 3 +(integer) 3 +127.0.0.1:6379[2]> lindex 'list_test' 1 +"2" +127.0.0.1:6379[2]> lrange 'list_test' 0 2 +1) "3" +2) "2" +3) "1" +127.0.0.1:6379[2]> linsert 'list_test' before a 'a' +(integer) -1 +127.0.0.1:6379[2]> linsert 'list_test' before 1 'a' +(integer) 4 +127.0.0.1:6379[2]> linsert 'list_test' before 'a' 'b' +(integer) 5 +127.0.0.1:6379[2]> rpop 'list_test' +"1" +127.0.0.1:6379[2]> lrem 'list_test' 1 1 +(integer) 0 +127.0.0.1:6379[2]> lrem 'list_test' 2 'a' +(integer) 1 +127.0.0.1:6379[2]> lrem 'list_test' 2 'b' +(integer) 1 +127.0.0.1:6379[2]> lpush 'list_test' 3 3 3 3 +(integer) 6 +127.0.0.1:6379[2]> lrem 'list_test' 3 2 +(integer) 1 +127.0.0.1:6379[2]> lrem 'list_test' 2 3 +(integer) 2 +127.0.0.1:6379[2]> sadd 'set_test' 1 +(integer) 1 +127.0.0.1:6379[2]> sadd 'set_test' 2 +(integer) 1 +127.0.0.1:6379[2]> smembers 'set_test' +1) "1" +2) "2" +127.0.0.1:6379[2]> sismember 'set_test' 2 +(integer) 1 +127.0.0.1:6379[2]> srandmember 'set_test' 2 +1) "1" +2) "2" +127.0.0.1:6379[2]> spop 'set_test' +"1" +127.0.0.1:6379[2]> zadd 'zse_test' 10 'a' +(integer) 1 +127.0.0.1:6379[2]> zadd 'zse_test' 1 'b' +(integer) 1 +127.0.0.1:6379[2]> zadd 'zse_test' 12 'c' +(integer) 1 +127.0.0.1:6379[2]> zcard 'zse_test' +(integer) 3 +127.0.0.1:6379[2]> zcount 'zse_test' 2 11 +(integer) 1 +127.0.0.1:6379[2]> zrangebyscore 'zse_test' 2 11 +1) "a" +127.0.0.1:6379[2]> zincrby 'zse_test' 1 'b' +"2" +127.0.0.1:6379[2]> zremrangebyrank 'zse_test' 1 2 +(integer) 2 +127.0.0.1:6379[2]> zremrangebyscore 'zse_test' 2 2 +(integer) 1 +127.0.0.1:6379[2]> +``` + -- Gitee