From 4ea51f34c9dd1f8d34e11f00edeb58040ab3244d Mon Sep 17 00:00:00 2001 From: wangyelou Date: Mon, 14 Jun 2021 22:01:23 +0800 Subject: [PATCH 1/5] =?UTF-8?q?selenium=20=E6=93=8D=E4=BD=9C=E5=B0=81?= =?UTF-8?q?=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 4 + pom.xml | 61 ++++++ src/main/java/cn/heiliu/selenium/Main.java | 32 ++++ .../heiliu/selenium/config/SetupConfig.java | 64 +++++++ .../heiliu/selenium/domain/SeleniumProxy.java | 56 ++++++ .../cn/heiliu/selenium/domain/SpecialUrl.java | 20 ++ .../selenium/domain/WebDriverGetResult.java | 40 ++++ .../selenium/domain/WebDriverStatus.java | 79 ++++++++ .../heiliu/selenium/enums/FileTypeEnum.java | 48 +++++ .../exception/DownFileTimeOutException.java | 12 ++ .../exception/UnsupportedMethodException.java | 12 ++ .../service/IWebDriverCreateService.java | 20 ++ .../service/IWebDriverGetService.java | 36 ++++ .../selenium/service/IWebDriverService.java | 60 ++++++ .../FirefoxWebDriverCreateServiceImpl.java | 89 +++++++++ .../impl/FirefoxWebDriverGetServiceImpl.java | 181 ++++++++++++++++++ .../impl/FirefoxWebDriverServiceImpl.java | 107 +++++++++++ src/main/resources/logback.xml | 30 +++ 18 files changed, 951 insertions(+) create mode 100644 pom.xml create mode 100644 src/main/java/cn/heiliu/selenium/Main.java create mode 100644 src/main/java/cn/heiliu/selenium/config/SetupConfig.java create mode 100644 src/main/java/cn/heiliu/selenium/domain/SeleniumProxy.java create mode 100644 src/main/java/cn/heiliu/selenium/domain/SpecialUrl.java create mode 100644 src/main/java/cn/heiliu/selenium/domain/WebDriverGetResult.java create mode 100644 src/main/java/cn/heiliu/selenium/domain/WebDriverStatus.java create mode 100644 src/main/java/cn/heiliu/selenium/enums/FileTypeEnum.java create mode 100644 src/main/java/cn/heiliu/selenium/exception/DownFileTimeOutException.java create mode 100644 src/main/java/cn/heiliu/selenium/exception/UnsupportedMethodException.java create mode 100644 src/main/java/cn/heiliu/selenium/service/IWebDriverCreateService.java create mode 100644 src/main/java/cn/heiliu/selenium/service/IWebDriverGetService.java create mode 100644 src/main/java/cn/heiliu/selenium/service/IWebDriverService.java create mode 100644 src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverCreateServiceImpl.java create mode 100644 src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverGetServiceImpl.java create mode 100644 src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverServiceImpl.java create mode 100644 src/main/resources/logback.xml diff --git a/.gitignore b/.gitignore index a1c2a23..05779ff 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,7 @@ # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml hs_err_pid* + +# idea +.idea/ +*.iml diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..302180d --- /dev/null +++ b/pom.xml @@ -0,0 +1,61 @@ + + + 4.0.0 + + cn.heiliu.selenium + selenium-client + 1.0-SNAPSHOT + + + 11 + true + + + + + + + cn.hutool + hutool-all + 5.6.3 + + + + org.seleniumhq.selenium + selenium-java + 3.141.59 + + + + org.projectlombok + lombok + 1.18.12 + + + + org.slf4j + slf4j-api + 1.8.0-beta2 + + + + ch.qos.logback + logback-core + 1.2.3 + + + + ch.qos.logback + logback-classic + 1.2.3 + + + + 　　org.slf4j + 　　slf4j-simple + 1.8.0-beta2 + + + \ No newline at end of file diff --git a/src/main/java/cn/heiliu/selenium/Main.java b/src/main/java/cn/heiliu/selenium/Main.java new file mode 100644 index 0000000..fed02dd --- /dev/null +++ b/src/main/java/cn/heiliu/selenium/Main.java @@ -0,0 +1,32 @@ +package cn.heiliu.selenium; + +import cn.heiliu.selenium.config.SetupConfig; +import cn.heiliu.selenium.domain.SeleniumProxy; +import cn.heiliu.selenium.domain.WebDriverGetResult; +import cn.heiliu.selenium.service.IWebDriverCreateService; +import cn.heiliu.selenium.service.IWebDriverGetService; +import cn.heiliu.selenium.service.IWebDriverService; +import cn.heiliu.selenium.service.impl.FirefoxWebDriverCreateServiceImpl; + +/** + * @author wangyelou + * @date 2021/6/14 + */ +public class Main { + + + public static void main(String[] args) { + + IWebDriverCreateService webDriverCreateService = new FirefoxWebDriverCreateServiceImpl(); + SetupConfig setupConfig = new SetupConfig("C:\\Program Files\\Mozilla Firefox\\firefox.exe", "D:\\work\\firefox-selenium\\geckodriver.exe", "D:\\work\\firefox-selenium\\firefox_proxy_auth.xpi"); + IWebDriverService webDriver = webDriverCreateService.createWebDriver(setupConfig); +// IWebDriverGetService fetch = webDriver.fetch("http://www.baidu.com/"); +// IWebDriverGetService fetch1 = webDriver.fetch("http://www.taobao.com/"); + +// WebDriverGetResult result = fetch.getResult(); + +// System.out.println(result); + + } + +} diff --git a/src/main/java/cn/heiliu/selenium/config/SetupConfig.java b/src/main/java/cn/heiliu/selenium/config/SetupConfig.java new file mode 100644 index 0000000..aef5d0c --- /dev/null +++ b/src/main/java/cn/heiliu/selenium/config/SetupConfig.java @@ -0,0 +1,64 @@ +package cn.heiliu.selenium.config; + +import lombok.Getter; + +import java.util.UUID; + +/** + * 浏览器启动配置 + * @author wangyelou + * @date 2021/6/14 + */ +@Getter +public class SetupConfig { + + /** + * 标志 + */ + private String flag; + + /** + * 浏览器执行路径 + */ + private String binPath; + + /** + * 引擎位置 + */ + private String enginePath; + + /** + * 是否加载图片 + */ + private boolean loadImage = false; + + /** + * 是否无头 + */ + private boolean headless = false; + + /** + * 文件下载位置 + */ + private String downDir; + + /** + * 代理认证插件位置 + */ + private String proxyAuthExt; + + public SetupConfig(String binPath, String enginePath, String proxyAuthExt) { + this.binPath = binPath; + this.enginePath = enginePath; + this.proxyAuthExt = proxyAuthExt; + this.flag = UUID.randomUUID().toString(); + } + + /** + * 获取下载目录 + * @return + */ + public String getDownDir() { + return flag; + } +} diff --git a/src/main/java/cn/heiliu/selenium/domain/SeleniumProxy.java b/src/main/java/cn/heiliu/selenium/domain/SeleniumProxy.java new file mode 100644 index 0000000..a520664 --- /dev/null +++ b/src/main/java/cn/heiliu/selenium/domain/SeleniumProxy.java @@ -0,0 +1,56 @@ +package cn.heiliu.selenium.domain; + +import cn.hutool.core.util.StrUtil; +import lombok.Getter; + +/** + * @author wangyelou + * @date 2021/6/14 + */ +@Getter +public class SeleniumProxy { + + private String schema = "http"; + + private String host; + + private int port; + + private String userName; + + private String passWord; + + public SeleniumProxy() { + } + + public SeleniumProxy(String schema, String host, int port, String userName, String passWord) { + this.schema = schema; + this.host = host; + this.port = port; + this.userName = userName; + this.passWord = passWord; + } + + public SeleniumProxy(String host, int port) { + this.host = host; + this.port = port; + } + + /** + * 是否为空 + * @return + */ + public boolean isEmpty() { + return StrUtil.hasEmpty(schema, host) || port <= 0; + } + + /** + * 是否认证 + * @return + */ + public boolean isAuth() { + return !isEmpty() && !StrUtil.hasEmpty(userName, passWord); + } + + +} diff --git a/src/main/java/cn/heiliu/selenium/domain/SpecialUrl.java b/src/main/java/cn/heiliu/selenium/domain/SpecialUrl.java new file mode 100644 index 0000000..68b3354 --- /dev/null +++ b/src/main/java/cn/heiliu/selenium/domain/SpecialUrl.java @@ -0,0 +1,20 @@ +package cn.heiliu.selenium.domain; + +/** + * 浏览器自定义url + * @author wangyelou + * @date 2021/6/14 + */ +public class SpecialUrl { + + /** + * 火狐配置页面 + */ + public static final String FIREFOX_CONFIG = "about:config"; + + /** + * 火狐空白页面 + */ + public static final String FIREFOX_BLANK = "about:blank"; + +} diff --git a/src/main/java/cn/heiliu/selenium/domain/WebDriverGetResult.java b/src/main/java/cn/heiliu/selenium/domain/WebDriverGetResult.java new file mode 100644 index 0000000..16edcb7 --- /dev/null +++ b/src/main/java/cn/heiliu/selenium/domain/WebDriverGetResult.java @@ -0,0 +1,40 @@ +package cn.heiliu.selenium.domain; + +import cn.heiliu.selenium.enums.FileTypeEnum; +import lombok.Getter; + +/** + * url访问结果 + * @author wangyelou + * @date 2021/6/14 + */ +@Getter +public class WebDriverGetResult { + + /** + * 页面内容 + */ + private byte[] content; + + /** + * 页面内容类型 + */ + private FileTypeEnum contentType; + + /** + * 是否是下载文件 + */ + private boolean isDownFile; + + /** + * 耗时 + */ + private long used; + + public WebDriverGetResult(byte[] content, FileTypeEnum contentType, boolean isDownFile, long used) { + this.content = content; + this.contentType = contentType; + this.isDownFile = isDownFile; + this.used = used; + } +} diff --git a/src/main/java/cn/heiliu/selenium/domain/WebDriverStatus.java b/src/main/java/cn/heiliu/selenium/domain/WebDriverStatus.java new file mode 100644 index 0000000..d56f193 --- /dev/null +++ b/src/main/java/cn/heiliu/selenium/domain/WebDriverStatus.java @@ -0,0 +1,79 @@ +package cn.heiliu.selenium.domain; + + +/** + * webdriver 状态 + * @author wangyelou + * @date 2021/6/14 + */ +public class WebDriverStatus { + + /** + * 是否有效 + */ + private boolean valid; + + /** + * 启动的时间戳 + */ + private long startUpTime; + + /** + * 空闲开始的时间戳 + */ + private long freeTime; + + /** + * 使用开始的时间戳 + */ + private long startUseTime; + + /** + * 是否在使用 + */ + private boolean used; + + /** + * 错误次数 + */ + private int failedNum = 0; + + public WebDriverStatus() { + startUpTime = System.currentTimeMillis(); + used = false; + valid = true; + } + + /** + * 开始使用 + */ + public void startUse() { + startUseTime = System.currentTimeMillis(); + used = true; + } + + /** + * 成功 + */ + public void success() { + freeTime = System.currentTimeMillis(); + failedNum = 0; + used = false; + } + + /** + * 失败 + */ + public void failed() { + freeTime = System.currentTimeMillis(); + failedNum ++; + used = false; + } + + /** + * 不可用 + */ + public void invalid() { + valid = false; + } +} diff --git a/src/main/java/cn/heiliu/selenium/enums/FileTypeEnum.java b/src/main/java/cn/heiliu/selenium/enums/FileTypeEnum.java new file mode 100644 index 0000000..c9fc39f --- /dev/null +++ b/src/main/java/cn/heiliu/selenium/enums/FileTypeEnum.java @@ -0,0 +1,48 @@ +package cn.heiliu.selenium.enums; + +import lombok.Getter; + +/** + * @author wangyelou + * @date 2021/6/14 + */ +public enum FileTypeEnum { + + /** + * html + */ + HTML("html"), + + /** + * pdf + */ + PDF("pdf"), + + /** + * excel xls + */ + XLS("xls"), + + /** + * excel xlsx + */ + XLSX("xlsx"), + + /** + * zip + */ + ZIP("zip"), + + /** + * 未知类型 + */ + UNKNOWN("unknown"); + + @Getter + private String value; + + private FileTypeEnum(String value) { + this.value = value; + } + +} diff --git a/src/main/java/cn/heiliu/selenium/exception/DownFileTimeOutException.java b/src/main/java/cn/heiliu/selenium/exception/DownFileTimeOutException.java new file mode 100644 index 0000000..45cb504 --- /dev/null +++ b/src/main/java/cn/heiliu/selenium/exception/DownFileTimeOutException.java @@ -0,0 +1,12 @@ +package cn.heiliu.selenium.exception; + +/** + * @author wangyelou + * @date 2021/6/14 + */ +public class DownFileTimeOutException extends RuntimeException { + + public DownFileTimeOutException(String message) { + super(message); + } +} diff --git a/src/main/java/cn/heiliu/selenium/exception/UnsupportedMethodException.java b/src/main/java/cn/heiliu/selenium/exception/UnsupportedMethodException.java new file mode 100644 index 0000000..db14a4d --- /dev/null +++ b/src/main/java/cn/heiliu/selenium/exception/UnsupportedMethodException.java @@ -0,0 +1,12 @@ +package cn.heiliu.selenium.exception; + +/** + * @author wangyelou + * @date 2021/6/14 + */ +public class UnsupportedMethodException extends RuntimeException { + + public UnsupportedMethodException(String message) { + super(message); + } +} diff --git a/src/main/java/cn/heiliu/selenium/service/IWebDriverCreateService.java b/src/main/java/cn/heiliu/selenium/service/IWebDriverCreateService.java new file mode 100644 index 0000000..eeadced --- /dev/null +++ b/src/main/java/cn/heiliu/selenium/service/IWebDriverCreateService.java @@ -0,0 +1,20 @@ +package cn.heiliu.selenium.service; + +import cn.heiliu.selenium.config.SetupConfig; + +/** + * webdriver 对象生成 + * @author wangyelou + * @date 2021/6/14 + */ +public interface IWebDriverCreateService { + + + /** + * 创建webdriver对象 + * @param setupConfig 启动配置 + * @return + */ + public IWebDriverService createWebDriver(SetupConfig setupConfig); + +} diff --git a/src/main/java/cn/heiliu/selenium/service/IWebDriverGetService.java b/src/main/java/cn/heiliu/selenium/service/IWebDriverGetService.java new file mode 100644 index 0000000..d7c926e --- /dev/null +++ b/src/main/java/cn/heiliu/selenium/service/IWebDriverGetService.java @@ -0,0 +1,36 @@ +package cn.heiliu.selenium.service; + +import cn.heiliu.selenium.domain.SeleniumProxy; +import cn.heiliu.selenium.domain.WebDriverGetResult; + +/** + * @author wangyelou + * @date 2021/6/14 + */ +public interface IWebDriverGetService { + + /** + * 访问url + */ + public void get(); + + /** + * 设置等待规则 + * @param xpath + */ + public void setWaitXpath(String xpath); + + /** + * 设置代理 + * @param proxy + */ + public void setProxy(SeleniumProxy proxy); + + /** + * 获取页面结果 + * @return + */ + public WebDriverGetResult getResult(); + + +} diff --git a/src/main/java/cn/heiliu/selenium/service/IWebDriverService.java b/src/main/java/cn/heiliu/selenium/service/IWebDriverService.java new file mode 100644 index 0000000..71b1080 --- /dev/null +++ b/src/main/java/cn/heiliu/selenium/service/IWebDriverService.java @@ -0,0 +1,60 @@ +package cn.heiliu.selenium.service; + +import cn.heiliu.selenium.domain.SeleniumProxy; +import cn.heiliu.selenium.domain.WebDriverStatus; +import cn.heiliu.selenium.exception.UnsupportedMethodException; +import org.openqa.selenium.WebDriver; + +/** + * @author wangyelou + * @date 2021/6/14 + */ +public interface IWebDriverService extends WebDriver { + + /** + * 获取浏览器标志 + */ + public String getFlag(); + + /** + * 访问url + * @param url + * @param proxy + * @param waitXpath + * @param timeOut + * @return + */ + public IWebDriverGetService fetch(String url, SeleniumProxy proxy, String waitXpath, long timeOut); + + public IWebDriverGetService fetch(String url, SeleniumProxy proxy); + + public IWebDriverGetService fetch(String url, String waitXpath, long timeOut); + + public IWebDriverGetService fetch(String url); + + /** + * 获取当前状态 + * @return + */ + public WebDriverStatus getStatus(); + + /** + * 禁止使用该方法 + * @return + */ + @Override + default String getPageSource() throws UnsupportedMethodException { + throw new UnsupportedMethodException("method not supported, please used getContent"); + } + + + /** + * 禁止使用get方法 + * @param url + */ + @Override + default void get(String url) { + throw new UnsupportedMethodException("method not supported, please used fetch"); + } + +} diff --git a/src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverCreateServiceImpl.java b/src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverCreateServiceImpl.java new file mode 100644 index 0000000..2477d46 --- /dev/null +++ b/src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverCreateServiceImpl.java @@ -0,0 +1,89 @@ +package cn.heiliu.selenium.service.impl; + +import cn.heiliu.selenium.config.SetupConfig; +import cn.heiliu.selenium.service.IWebDriverCreateService; +import cn.heiliu.selenium.service.IWebDriverService; +import lombok.extern.slf4j.Slf4j; +import org.openqa.selenium.firefox.FirefoxDriver; +import org.openqa.selenium.firefox.FirefoxOptions; + +import java.io.File; + +/** + * @author wangyelou + * @date 2021/6/14 + */ +@Slf4j +public class FirefoxWebDriverCreateServiceImpl implements IWebDriverCreateService { + + private static final String GECKO_PATH = "webdriver.gecko.driver"; + + @Override + public IWebDriverService createWebDriver(SetupConfig setupConfig) { + FirefoxOptions surrogate = getDefaultOption(setupConfig); + + // 启动 + FirefoxWebDriverServiceImpl webDriver = new FirefoxWebDriverServiceImpl(surrogate, setupConfig); + + // 代理认证 + webDriver.installExtension(new File(setupConfig.getProxyAuthExt()).toPath()); + + log.info("create firefox[{}]", webDriver.getFlag()); + return webDriver; + } + + /** + * 默认配置设置 + * @param setupConfig + * @return + */ + private FirefoxOptions getDefaultOption(SetupConfig setupConfig) { + // 注入环境变量 + if (setupConfig.getEnginePath() != null) { + System.setProperty(GECKO_PATH, setupConfig.getEnginePath()); + } + if (setupConfig.getBinPath() != null) { + System.setProperty(FirefoxDriver.SystemProperty.BROWSER_BINARY, setupConfig.getBinPath()); + } + + FirefoxOptions surrogate = new FirefoxOptions(); + + // 禁用pdf.js + surrogate.addPreference("pdfjs.disabled", true); + surrogate.addPreference("pdfjs.enabledCache.state", false); + + // 不打开未知类型 + surrogate.addPreference("browser.helperApps.alwaysAsk.force", false); + + // 不弹出警告框 + surrogate.addPreference("browser.download.manager.alertOnEXEopen", false); + surrogate.addPreference("browser.download.manager.focusWhenStarting", false); + surrogate.addPreference("browser.download.manager.useWindow", false); + surrogate.addPreference("browser.download.manager.showAlertOnComplete", true); + surrogate.addPreference("browser.download.manager.closeWhenDone", false); + surrogate.addPreference("browser.aboutConfig.showWarning", false); + + // 不联其他网站 + surrogate.addPreference("network.captive-portal-service.enabled", false); + + // 不加载图片 + surrogate.addPreference("permissions.default.image", setupConfig.isLoadImage() ? 1 : 2); + + // 不输出geckodriver 日志 + System.setProperty(FirefoxDriver.SystemProperty.BROWSER_LOGFILE, "/dev/null"); + + // 自动下载文件类型 + surrogate.addPreference("browser.download.folderList", 2); + surrogate.addPreference("browser.download.dir", setupConfig.getDownDir()); + surrogate.addPreference("browser.download.useDownloadDir", true); + surrogate.addPreference("browser.download.manager.showWhenStarting", false); + surrogate.addPreference("browser.helperApps.neverAsk.saveToDisk", "application/octet-stream, application/x-001, text/h323, drawing/907, audio/x-mei-aac, audio/aiff, text/asa, text/asp, audio/basic, application/vnd.adobe.workflow, application/x-bmp, application/x-c4t, application/x-cals, application/x-netcdf, application/x-cel, application/x-g4, application/x-cit, application/x-cmx, application/pkix-crl, application/x-csi, application/x-cut, application/x-dbm, application/x-x509-ca-cert, application/x-dib, application/msword, application/x-drw, Model/vnd.dwf, application/x-dwg, application/x-dxf, application/x-emf, application/x-ps, application/x-ebx, image/fax, application/fractals, application/x-frm, application/x-gbr, image/gif, application/x-gp4, application/x-hmr, application/x-hpl, application/x-hrf, text/x-component, image/x-icon, application/x-iff, application/x-igs, application/x-img, application/x-internet-signup, java/*, image/jpeg, application/x-jpg, application/x-laplayer-reg, audio/x-liquid-secure, audio/x-la-lms, application/x-ltr, video/x-mpeg, video/mpeg4, application/x-troff-man, application/msaccess, application/x-shockwave-flash, message/rfc822, audio/mid, application/x-mil, audio/x-musicnet-download, application/x-javascript, audio/mp1, video/mpeg, application/vnd.ms-project, video/mpg, audio/rn-mpeg, image/pnetvue, application/x-out, application/x-pkcs12, application/pkcs7-mime, application/x-pkcs7-certreqresp, application/x-pc5, application/x-pcl, application/pdf, application/vnd.adobe.pdx, application/x-pgl, application/vnd.ms-pki.pko, application/x-plt, application/x-png, application/vnd.ms-powerpoint, application/x-ppt, application/pics-rules, application/x-prt, application/postscript, audio/vnd.rn-realaudio, application/x-ras, application/x-red, application/vnd.rn-realsystem-rjs, application/x-rlc, application/vnd.rn-realmedia, audio/x-pn-realaudio, application/vnd.rn-realmedia-secure, application/vnd.rn-realsystem-rmx, image/vnd.rn-realpix, application/vnd.rn-rsml, video/vnd.rn-realvideo, application/x-sat, application/x-sdw, application/x-slb, drawing/x-slk, application/smil, text/plain, application/futuresplash, application/streamingmedia, application/vnd.ms-pki.stl, application/x-sty, application/x-tg4, image/tiff, drawing/x-top, application/x-icq, text/x-vcard, application/vnd.visio, application/x-vpeg005, application/x-vsd, audio/wav, application/x-wb1, application/x-wb3, application/x-wk4, application/x-wks, audio/x-ms-wma, application/x-wmf, video/x-ms-wmv, application/x-ms-wmz, application/x-wpd, application/vnd.ms-wpl, application/x-wr1, application/x-wrk, application/x-ws, application/vnd.adobe.xdp, application/vnd.adobe.xfd, application/x-xls, application/x-xwd, application/vnd.symbian.install, application/x-x_t, application/vnd.android.package-archive, application/x-301, application/x-906, application/x-a11, application/x-anv, video/x-ms-asf, video/avi, application/x-bot, application/x-c90, application/vnd.ms-pki.seccat, application/x-cdr, application/x-cgm, application/x-cmp, application/x-cot, application/x-dbf, application/x-dbx, application/x-dcx, application/x-dgn, application/x-msdownload, application/x-dwf, application/x-dxb, application/vnd.adobe.edn, application/x-epi, application/vnd.fdf, application/x-, application/x-gl2, application/x-hgl, application/x-hpgl, application/mac-binhex40, application/hta, text/webviewhtml, application/x-icb, application/x-ico, application/x-iphone, video/x-ivf, application/x-jpe, audio/x-liquid-file, application/x-latex, application/x-lbm, audio/mpegurl, application/x-mac, application/x-mdb, application/x-mi, audio/x-musicnet-stream, video/x-sgi-movie, audio/mp2, audio/mp3, video/x-mpg, application/x-mmxp, application/x-nrf, text/x-ms-odc, application/pkcs10, application/x-pkcs7-certificates, application/pkcs7-signature, application/x-pci, application/x-pcx, application/x-pic, application/x-perl, audio/scpls, image/png, application/x-ppm, application/x-pr, application/x-prn, application/x-ptn, text/vnd.rn-realtext3d, application/rat-file, application/vnd.rn-recording, application/x-rgb, application/vnd.rn-realsystem-rjt, application/x-rle, application/vnd.adobe.rmf, application/vnd.rn-realsystem-rmj, application/vnd.rn-rn_music_package, application/vnd.rn-realmedia-vbr, application/vnd.rn-realplayer, audio/x-pn-realaudio-plugin, text/vnd.rn-realtext, application/x-rtf, application/x-sam, application/sdp, application/x-stuffit, application/x-sld, application/x-smk, application/vnd.ms-pki.certstore, application/x-tdf, application/x-tga, application/x-tif, application/x-bittorrent, text/iuls, application/x-vda, application/x-vst, audio/x-ms-wax, application/x-wb2, image/vnd.wap.wbmp, application/x-wk3, application/x-wkq, video/x-ms-wm, application/x-ms-wmd, text/vnd.wap.wml, video/x-ms-wmx, application/x-wp6, application/x-wpg, application/x-wq1, application/x-wri, text/scriptlet, video/x-ms-wvx, application/vnd.adobe.xfdf, application/vnd.ms-excel, application/x-xlw, application/x-x_b, application/vnd.iphone, application/x-silverlight-app, application/msexcel, application/msword, application/zip, application/vnd.openxmlformats-officedocument.wordprocessingml.document, application/x-zip-compressed"); + + // 是否无头 + if (setupConfig.isHeadless()) { + surrogate.addArguments("--headless"); + } + + return surrogate; + } +} diff --git a/src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverGetServiceImpl.java b/src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverGetServiceImpl.java new file mode 100644 index 0000000..abcdea9 --- /dev/null +++ b/src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverGetServiceImpl.java @@ -0,0 +1,181 @@ +package cn.heiliu.selenium.service.impl; + +import cn.heiliu.selenium.domain.SeleniumProxy; +import cn.heiliu.selenium.domain.SpecialUrl; +import cn.heiliu.selenium.domain.WebDriverGetResult; +import cn.heiliu.selenium.enums.FileTypeEnum; +import cn.heiliu.selenium.exception.DownFileTimeOutException; +import cn.heiliu.selenium.service.IWebDriverGetService; +import cn.hutool.core.io.FileUtil; +import cn.hutool.core.thread.ThreadUtil; +import cn.hutool.core.util.StrUtil; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.openqa.selenium.By; +import org.openqa.selenium.NoSuchElementException; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +/** + * @author wangyelou + * @date 2021/6/14 + */ +@Slf4j +public class FirefoxWebDriverGetServiceImpl implements IWebDriverGetService { + + private String downDir; + private FirefoxWebDriverServiceImpl webDriver; + private String url; + private long timeOut; + @Setter + private String waitXpath; + @Setter + private SeleniumProxy proxy; + private File downFile; + private long startTime = System.currentTimeMillis(); + + private static final Map EXT_RELATIONS = new HashMap<>(16); + + static { + EXT_RELATIONS.put("pdf", FileTypeEnum.PDF); + EXT_RELATIONS.put("zip", FileTypeEnum.ZIP); + EXT_RELATIONS.put("xls", FileTypeEnum.XLS); + EXT_RELATIONS.put("xlsx", FileTypeEnum.XLSX); + } + + public FirefoxWebDriverGetServiceImpl() { + } + + public FirefoxWebDriverGetServiceImpl(FirefoxWebDriverServiceImpl webDriver, String downDir, String url, long timeOut) { + this.downDir = downDir; + this.webDriver = webDriver; + this.url = url; + this.timeOut = timeOut; + } + + @Override + public void get() { + // 设置代理 + proxySet(); + + // 清空当前标签页内容 + webDriver.get(SpecialUrl.FIREFOX_BLANK); + String currentWindow = webDriver.getWindowHandle(); + + // 关闭除当前标签的所有标签 + webDriver.getWindowHandles().forEach(window -> { + if (!window.equals(currentWindow)) { + webDriver.switchTo().window(window); + webDriver.close(); + } + }); + + // 删除所有下载文件 + FileUtil.clean(downDir); + + // 执行js， 跳转页面 + webDriver.switchTo().window(currentWindow); + webDriver.executeScript("window.open(\"" + formatUrl(url) + "\")"); + + // 切换标签 + webDriver.getWindowHandles().forEach(window -> { + if (!window.equals(currentWindow)) { + webDriver.switchTo().window(window); + } + }); + + // 结果处理 + dealResult(); + } + + /** + * 下载结果处理 + */ + private void dealResult() { + long startTime = System.currentTimeMillis(); + boolean htmlFlag = false; + boolean downFlag = false; + while (!htmlFlag && !downFlag && System.currentTimeMillis() - startTime < timeOut) { + // html页面元素检查 + if (!StrUtil.hasEmpty(waitXpath) && !webDriver.findElements(By.xpath(waitXpath)).isEmpty()) { + log.info("find element with xpath: {}", waitXpath); + htmlFlag = true; + } + + // 下载文件检查 + if (FileUtil.exist(downDir)) { + log.info("file downloading..."); + // 检测到文件下载，则等待下载完成 + while (FileUtil.ls(downDir).length > 1 && System.currentTimeMillis() - startTime < timeOut) { + ThreadUtil.sleep(1000); + } + + // 获取下载文件路径 + for (File file : FileUtil.ls(downDir)) { + downFile = file; + log.info("get down file: {}", downFile.getName()); + } + downFlag = true; + } + + ThreadUtil.sleep(500); + } + + if (System.currentTimeMillis() - startTime > timeOut) { + if (downFlag) { + throw new DownFileTimeOutException("web driver download file time out " + timeOut); + } else { + throw new NoSuchElementException("no such element " + waitXpath); + } + } + + + } + + /** + * 获取结果 + * @return + */ + @Override + public WebDriverGetResult getResult() { + return new WebDriverGetResult( + FileUtil.isFile(downFile) ? FileUtil.readBytes(downFile) : webDriver.getPageSource().getBytes(), + FileUtil.isFile(downFile) ? EXT_RELATIONS.getOrDefault(FileUtil.extName(downFile), FileTypeEnum.UNKNOWN) : FileTypeEnum.HTML, + downFile != null, + System.currentTimeMillis() - startTime + ); + } + + /** + * 利用js设置代理 + */ + private void proxySet() { + StringBuilder js = new StringBuilder(); + js.append("var prefs = Components.classes['@mozilla.org/preferences-service;1'].getService(Components.interfaces.nsIPrefBranch);"); + js.append(String.format("prefs.setIntPref('network.proxy.type', %s);", proxy != null && !proxy.isEmpty() ? 1 : 0)); + + if (proxy != null && !proxy.isEmpty()) { + js.append(String.format("prefs.setCharPref('network.proxy.http', '%s');", proxy.getHost())); + js.append(String.format("prefs.setIntPref('network.proxy.http_port', %s);", proxy.getPort())); + js.append(String.format("prefs.setCharPref('network.proxy.ssl', '%s');", proxy.getHost())); + js.append(String.format("prefs.setIntPref('network.proxy.ssl_port', %s);", proxy.getPort())); + } + webDriver.get(SpecialUrl.FIREFOX_CONFIG); + webDriver.executeScript(js.toString()); + } + + /** + * 根据firefox auth 插件要求配置自定义认证信息 + * @param url + * @return + */ + private String formatUrl(String url) { + if (proxy != null && proxy.isAuth()) { + return String.format("%s~%s:%s", url, proxy.getUserName(), proxy.getPassWord()); + } + return url; + } + +} diff --git a/src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverServiceImpl.java b/src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverServiceImpl.java new file mode 100644 index 0000000..8b711ba --- /dev/null +++ b/src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverServiceImpl.java @@ -0,0 +1,107 @@ +package cn.heiliu.selenium.service.impl; + +import cn.heiliu.selenium.config.SetupConfig; +import cn.heiliu.selenium.domain.SeleniumProxy; +import cn.heiliu.selenium.domain.WebDriverStatus; +import cn.heiliu.selenium.service.IWebDriverGetService; +import cn.heiliu.selenium.service.IWebDriverService; +import com.google.common.collect.ImmutableMap; +import org.openqa.selenium.firefox.FirefoxDriver; +import org.openqa.selenium.firefox.FirefoxOptions; + +import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author wangyelou + * @date 2021/6/14 + */ +public class FirefoxWebDriverServiceImpl extends FirefoxDriver implements IWebDriverService { + + private static final String INSTALL_EXTENSION = "installExtension"; + private static final long DEFAULT_TIME_OUT = 10000; + private static final String DEFAULT_WAIT_XPATH = "//div | //p | //table | //tr | //td | //span"; + private static final String FIREFOX_SPECIAL_URL_PREFIX = "about:"; + private static AtomicInteger num = new AtomicInteger(1); + + /** + * 启动配置 + */ + private SetupConfig setupConfig; + + /** + * 浏览器标志 + */ + private String flag; + + /** + * 状态记录 + */ + private WebDriverStatus status; + + public FirefoxWebDriverServiceImpl(FirefoxOptions firefoxOptions, SetupConfig setupConfig) { + super(firefoxOptions); + this.setupConfig = setupConfig; + this.flag = "firefox-" + num.getAndIncrement(); + num.compareAndSet(10000, 1); + status = new WebDriverStatus(); + } + + /** + * 安装临时插件 + * temporary 是否是临时插件 + * @param path 插件位置，绝对路径 + * @return + */ + @Override + public String installExtension(Path path) { + return (String) execute(INSTALL_EXTENSION, + ImmutableMap.of("path", path.toAbsolutePath().toString(), + "temporary", true)).getValue(); + } + + @Override + public String getFlag() { + return flag; + } + + @Override + public IWebDriverGetService fetch(String url, SeleniumProxy proxy, String waitXpath, long timeOut) { + FirefoxWebDriverGetServiceImpl webGet = new FirefoxWebDriverGetServiceImpl(this, setupConfig.getDownDir(), url, timeOut); + webGet.setWaitXpath(waitXpath); + if (proxy != null && !proxy.isEmpty()) { + webGet.setProxy(proxy); + } + webGet.get(); + return webGet; + } + + @Override + public IWebDriverGetService fetch(String url, SeleniumProxy proxy) { + return fetch(url, proxy, DEFAULT_WAIT_XPATH, DEFAULT_TIME_OUT); + } + + @Override + public IWebDriverGetService fetch(String url, String waitXpath, long timeOut) { + return fetch(url, null, waitXpath, timeOut); + } + + /** + * 如果访问配置页面，则使用原来的get + * @param url + */ + @Override + public IWebDriverGetService fetch(String url) { + if (url.indexOf(FIREFOX_SPECIAL_URL_PREFIX) == 0) { + super.get(url); + return new FirefoxWebDriverGetServiceImpl(); + } else { + return fetch(url, null, DEFAULT_WAIT_XPATH, DEFAULT_TIME_OUT); + } + } + + @Override + public WebDriverStatus getStatus() { + return status; + } +} diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml new file mode 100644 index 0000000..490e425 --- /dev/null +++ b/src/main/resources/logback.xml @@ -0,0 +1,30 @@ + + + + + + + + %d %p (%file:%line\) - [%thread] [%X{key}] %m%n + UTF-8 + + + + + ${LOG_HOME}/server.log + + ${LOG_HOME}/server.%d.log + 7 + + + {"logtime": "%d", "level": "%level","thread": "%thread","class_name": "%class","line_number": "%line","message": "%message"}%n + UTF-8 + + + + + + + + + \ No newline at end of file -- Gitee From d415dc3bbc83d6e2aec5f0dbc42f1529f449b0fb Mon Sep 17 00:00:00 2001 From: wangyelou Date: Tue, 15 Jun 2021 00:14:03 +0800 Subject: [PATCH 2/5] =?UTF-8?q?selenium=E7=AE=A1=E7=90=86=E6=B1=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../{Main.java => SeleniumManager.java} | 37 ++--- .../heiliu/selenium/config/SetupConfig.java | 25 ++-- .../selenium/domain/WebDriverStatus.java | 24 ++-- .../exception/NoWebDriverException.java | 12 ++ .../service/ISeleniumPoolService.java | 33 +++++ .../selenium/service/IWebDriverService.java | 6 +- .../impl/FirefoxWebDriverGetServiceImpl.java | 14 +- .../impl/FirefoxWebDriverServiceImpl.java | 34 +++-- .../service/impl/SeleniumPoolServiceImpl.java | 126 ++++++++++++++++++ .../heiliu/selenium/GetSingleWebDriver.java | 24 ++++ .../heiliu/selenium/GetWebDriverFromPool.java | 40 ++++++ .../META-INF/SeleniumClient.kotlin_module | Bin 0 -> 16 bytes target/classes/logback.xml | 30 +++++ 13 files changed, 353 insertions(+), 52 deletions(-) rename src/main/java/cn/heiliu/selenium/{Main.java => SeleniumManager.java} (35%) create mode 100644 src/main/java/cn/heiliu/selenium/exception/NoWebDriverException.java create mode 100644 src/main/java/cn/heiliu/selenium/service/ISeleniumPoolService.java create mode 100644 src/main/java/cn/heiliu/selenium/service/impl/SeleniumPoolServiceImpl.java create mode 100644 src/test/java/cn/heiliu/selenium/GetSingleWebDriver.java create mode 100644 src/test/java/cn/heiliu/selenium/GetWebDriverFromPool.java create mode 100644 target/classes/META-INF/SeleniumClient.kotlin_module create mode 100644 target/classes/logback.xml diff --git a/src/main/java/cn/heiliu/selenium/Main.java b/src/main/java/cn/heiliu/selenium/SeleniumManager.java similarity index 35% rename from src/main/java/cn/heiliu/selenium/Main.java rename to src/main/java/cn/heiliu/selenium/SeleniumManager.java index fed02dd..b34b56e 100644 --- a/src/main/java/cn/heiliu/selenium/Main.java +++ b/src/main/java/cn/heiliu/selenium/SeleniumManager.java @@ -1,32 +1,37 @@ package cn.heiliu.selenium; import cn.heiliu.selenium.config.SetupConfig; -import cn.heiliu.selenium.domain.SeleniumProxy; -import cn.heiliu.selenium.domain.WebDriverGetResult; +import cn.heiliu.selenium.service.ISeleniumPoolService; import cn.heiliu.selenium.service.IWebDriverCreateService; -import cn.heiliu.selenium.service.IWebDriverGetService; import cn.heiliu.selenium.service.IWebDriverService; import cn.heiliu.selenium.service.impl.FirefoxWebDriverCreateServiceImpl; +import cn.heiliu.selenium.service.impl.FirefoxWebDriverServiceImpl; +import cn.heiliu.selenium.service.impl.SeleniumPoolServiceImpl; /** * @author wangyelou * @date 2021/6/14 */ -public class Main { - - - public static void main(String[] args) { +public class SeleniumManager { + + /** + * 创建firefox管理池 + * @param setupConfig + * @return + */ + public static ISeleniumPoolService createFirefoxSeleniumPool(SetupConfig setupConfig) { + return new SeleniumPoolServiceImpl(new FirefoxWebDriverCreateServiceImpl(), setupConfig); + } + /** + * 生成firefox浏览器 + * @param setupConfig + * @return + */ + public static IWebDriverService createFirefoxSelenium(SetupConfig setupConfig) { IWebDriverCreateService webDriverCreateService = new FirefoxWebDriverCreateServiceImpl(); - SetupConfig setupConfig = new SetupConfig("C:\\Program Files\\Mozilla Firefox\\firefox.exe", "D:\\work\\firefox-selenium\\geckodriver.exe", "D:\\work\\firefox-selenium\\firefox_proxy_auth.xpi"); - IWebDriverService webDriver = webDriverCreateService.createWebDriver(setupConfig); -// IWebDriverGetService fetch = webDriver.fetch("http://www.baidu.com/"); -// IWebDriverGetService fetch1 = webDriver.fetch("http://www.taobao.com/"); - -// WebDriverGetResult result = fetch.getResult(); - -// System.out.println(result); - + return webDriverCreateService.createWebDriver(setupConfig); } + } diff --git a/src/main/java/cn/heiliu/selenium/config/SetupConfig.java b/src/main/java/cn/heiliu/selenium/config/SetupConfig.java index aef5d0c..4e133fb 100644 --- a/src/main/java/cn/heiliu/selenium/config/SetupConfig.java +++ b/src/main/java/cn/heiliu/selenium/config/SetupConfig.java @@ -1,5 +1,6 @@ package cn.heiliu.selenium.config; +import lombok.Builder; import lombok.Getter; import java.util.UUID; @@ -9,13 +10,27 @@ import java.util.UUID; * @author wangyelou * @date 2021/6/14 */ +@Builder @Getter public class SetupConfig { + /** + * 池大小 + */ + @Builder.Default + private int poolSize = 10; + + /** + * 最大连续失败次数 + */ + @Builder.Default + private int maxFailedNum = 10; + /** * 标志 */ - private String flag; + @Builder.Default + private String flag = UUID.randomUUID().toString(); /** * 浏览器执行路径 @@ -47,13 +62,6 @@ public class SetupConfig { */ private String proxyAuthExt; - public SetupConfig(String binPath, String enginePath, String proxyAuthExt) { - this.binPath = binPath; - this.enginePath = enginePath; - this.proxyAuthExt = proxyAuthExt; - this.flag = UUID.randomUUID().toString(); - } - /** * 获取下载目录 * @return @@ -61,4 +69,5 @@ public class SetupConfig { public String getDownDir() { return flag; } + } diff --git a/src/main/java/cn/heiliu/selenium/domain/WebDriverStatus.java b/src/main/java/cn/heiliu/selenium/domain/WebDriverStatus.java index d56f193..5ce5016 100644 --- a/src/main/java/cn/heiliu/selenium/domain/WebDriverStatus.java +++ b/src/main/java/cn/heiliu/selenium/domain/WebDriverStatus.java @@ -1,6 +1,8 @@ package cn.heiliu.selenium.domain; +import lombok.Getter; + /** * webdriver 状态 * @author wangyelou @@ -11,31 +13,31 @@ public class WebDriverStatus { /** * 是否有效 */ + @Getter private boolean valid; /** * 启动的时间戳 */ + @Getter private long startUpTime; - /** - * 空闲开始的时间戳 - */ - private long freeTime; - /** * 使用开始的时间戳 */ + @Getter private long startUseTime; /** * 是否在使用 */ + @Getter private boolean used; /** * 错误次数 */ + @Getter private int failedNum = 0; public WebDriverStatus() { @@ -48,25 +50,29 @@ public class WebDriverStatus { * 开始使用 */ public void startUse() { - startUseTime = System.currentTimeMillis(); used = true; + startUseTime = System.currentTimeMillis(); } /** * 成功 */ public void success() { - freeTime = System.currentTimeMillis(); failedNum = 0; - used = false; } /** * 失败 */ public void failed() { - freeTime = System.currentTimeMillis(); failedNum ++; + } + + + /** + * 停止使用 + */ + public void release() { used = false; } diff --git a/src/main/java/cn/heiliu/selenium/exception/NoWebDriverException.java b/src/main/java/cn/heiliu/selenium/exception/NoWebDriverException.java new file mode 100644 index 0000000..2c83022 --- /dev/null +++ b/src/main/java/cn/heiliu/selenium/exception/NoWebDriverException.java @@ -0,0 +1,12 @@ +package cn.heiliu.selenium.exception; + +/** + * @author wangyelou + * @date 2021/6/14 + */ +public class NoWebDriverException extends RuntimeException { + + public NoWebDriverException(String message) { + super(message); + } +} diff --git a/src/main/java/cn/heiliu/selenium/service/ISeleniumPoolService.java b/src/main/java/cn/heiliu/selenium/service/ISeleniumPoolService.java new file mode 100644 index 0000000..35bcdf9 --- /dev/null +++ b/src/main/java/cn/heiliu/selenium/service/ISeleniumPoolService.java @@ -0,0 +1,33 @@ +package cn.heiliu.selenium.service; + +import org.openqa.selenium.TimeoutException; + +import java.util.Set; + +/** + * @author wangyelou + * @date 2021/6/14 + */ +public interface ISeleniumPoolService { + + /** + * 池中获取有效的webdriver + * @param timeout + * @return + * @throws TimeoutException + */ + public IWebDriverService getWebDriver(long timeout) throws TimeoutException; + + /** + * 释放webdriver + * @param webDriver + */ + public void release(IWebDriverService webDriver); + + /** + * 返回当前可用webdriver数量 + * @return + */ + public int getValidNum(); + +} diff --git a/src/main/java/cn/heiliu/selenium/service/IWebDriverService.java b/src/main/java/cn/heiliu/selenium/service/IWebDriverService.java index 71b1080..77d4874 100644 --- a/src/main/java/cn/heiliu/selenium/service/IWebDriverService.java +++ b/src/main/java/cn/heiliu/selenium/service/IWebDriverService.java @@ -21,14 +21,14 @@ public interface IWebDriverService extends WebDriver { * @param url * @param proxy * @param waitXpath - * @param timeOut + * @param timeout * @return */ - public IWebDriverGetService fetch(String url, SeleniumProxy proxy, String waitXpath, long timeOut); + public IWebDriverGetService fetch(String url, SeleniumProxy proxy, String waitXpath, long timeout); public IWebDriverGetService fetch(String url, SeleniumProxy proxy); - public IWebDriverGetService fetch(String url, String waitXpath, long timeOut); + public IWebDriverGetService fetch(String url, String waitXpath, long timeout); public IWebDriverGetService fetch(String url); diff --git a/src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverGetServiceImpl.java b/src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverGetServiceImpl.java index abcdea9..413c345 100644 --- a/src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverGetServiceImpl.java +++ b/src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverGetServiceImpl.java @@ -28,7 +28,7 @@ public class FirefoxWebDriverGetServiceImpl implements IWebDriverGetService { private String downDir; private FirefoxWebDriverServiceImpl webDriver; private String url; - private long timeOut; + private long timeout; @Setter private String waitXpath; @Setter @@ -48,11 +48,11 @@ public class FirefoxWebDriverGetServiceImpl implements IWebDriverGetService { public FirefoxWebDriverGetServiceImpl() { } - public FirefoxWebDriverGetServiceImpl(FirefoxWebDriverServiceImpl webDriver, String downDir, String url, long timeOut) { + public FirefoxWebDriverGetServiceImpl(FirefoxWebDriverServiceImpl webDriver, String downDir, String url, long timeout) { this.downDir = downDir; this.webDriver = webDriver; this.url = url; - this.timeOut = timeOut; + this.timeout = timeout; } @Override @@ -97,7 +97,7 @@ public class FirefoxWebDriverGetServiceImpl implements IWebDriverGetService { long startTime = System.currentTimeMillis(); boolean htmlFlag = false; boolean downFlag = false; - while (!htmlFlag && !downFlag && System.currentTimeMillis() - startTime < timeOut) { + while (!htmlFlag && !downFlag && System.currentTimeMillis() - startTime < timeout) { // html页面元素检查 if (!StrUtil.hasEmpty(waitXpath) && !webDriver.findElements(By.xpath(waitXpath)).isEmpty()) { log.info("find element with xpath: {}", waitXpath); @@ -108,7 +108,7 @@ public class FirefoxWebDriverGetServiceImpl implements IWebDriverGetService { if (FileUtil.exist(downDir)) { log.info("file downloading..."); // 检测到文件下载，则等待下载完成 - while (FileUtil.ls(downDir).length > 1 && System.currentTimeMillis() - startTime < timeOut) { + while (FileUtil.ls(downDir).length > 1 && System.currentTimeMillis() - startTime < timeout) { ThreadUtil.sleep(1000); } @@ -123,9 +123,9 @@ public class FirefoxWebDriverGetServiceImpl implements IWebDriverGetService { ThreadUtil.sleep(500); } - if (System.currentTimeMillis() - startTime > timeOut) { + if (System.currentTimeMillis() - startTime > timeout) { if (downFlag) { - throw new DownFileTimeOutException("web driver download file time out " + timeOut); + throw new DownFileTimeOutException("web driver download file time out " + timeout); } else { throw new NoSuchElementException("no such element " + waitXpath); } diff --git a/src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverServiceImpl.java b/src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverServiceImpl.java index 8b711ba..4236ca6 100644 --- a/src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverServiceImpl.java +++ b/src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverServiceImpl.java @@ -3,6 +3,7 @@ package cn.heiliu.selenium.service.impl; import cn.heiliu.selenium.config.SetupConfig; import cn.heiliu.selenium.domain.SeleniumProxy; import cn.heiliu.selenium.domain.WebDriverStatus; +import cn.heiliu.selenium.exception.DownFileTimeOutException; import cn.heiliu.selenium.service.IWebDriverGetService; import cn.heiliu.selenium.service.IWebDriverService; import com.google.common.collect.ImmutableMap; @@ -66,14 +67,23 @@ public class FirefoxWebDriverServiceImpl extends FirefoxDriver implements IWebDr } @Override - public IWebDriverGetService fetch(String url, SeleniumProxy proxy, String waitXpath, long timeOut) { - FirefoxWebDriverGetServiceImpl webGet = new FirefoxWebDriverGetServiceImpl(this, setupConfig.getDownDir(), url, timeOut); - webGet.setWaitXpath(waitXpath); - if (proxy != null && !proxy.isEmpty()) { - webGet.setProxy(proxy); + public IWebDriverGetService fetch(String url, SeleniumProxy proxy, String waitXpath, long timeout) { + try { + FirefoxWebDriverGetServiceImpl webGet = new FirefoxWebDriverGetServiceImpl(this, setupConfig.getDownDir(), url, timeout); + webGet.setWaitXpath(waitXpath); + if (proxy != null && !proxy.isEmpty()) { + webGet.setProxy(proxy); + } + webGet.get(); + return webGet; + } catch (DownFileTimeOutException e) { + // 浏览器文件下载超时，但是浏览器本身还在下载，需要强制关闭浏览器 + this.quit(); + throw e; + } catch (Exception e) { + status.failed(); + throw e; } - webGet.get(); - return webGet; } @Override @@ -82,8 +92,8 @@ public class FirefoxWebDriverServiceImpl extends FirefoxDriver implements IWebDr } @Override - public IWebDriverGetService fetch(String url, String waitXpath, long timeOut) { - return fetch(url, null, waitXpath, timeOut); + public IWebDriverGetService fetch(String url, String waitXpath, long timeout) { + return fetch(url, null, waitXpath, timeout); } /** @@ -104,4 +114,10 @@ public class FirefoxWebDriverServiceImpl extends FirefoxDriver implements IWebDr public WebDriverStatus getStatus() { return status; } + + @Override + public void quit() { + status.invalid(); + super.quit(); + } } diff --git a/src/main/java/cn/heiliu/selenium/service/impl/SeleniumPoolServiceImpl.java b/src/main/java/cn/heiliu/selenium/service/impl/SeleniumPoolServiceImpl.java new file mode 100644 index 0000000..788306d --- /dev/null +++ b/src/main/java/cn/heiliu/selenium/service/impl/SeleniumPoolServiceImpl.java @@ -0,0 +1,126 @@ +package cn.heiliu.selenium.service.impl; + +import cn.heiliu.selenium.config.SetupConfig; +import cn.heiliu.selenium.domain.WebDriverStatus; +import cn.heiliu.selenium.exception.NoWebDriverException; +import cn.heiliu.selenium.service.ISeleniumPoolService; +import cn.heiliu.selenium.service.IWebDriverCreateService; +import cn.heiliu.selenium.service.IWebDriverService; +import cn.hutool.core.thread.ThreadFactoryBuilder; +import cn.hutool.core.thread.ThreadUtil; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.openqa.selenium.TimeoutException; + +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Random; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * selenium 连接池实现 + * @author wangyelou + * @date 2021/6/14 + */ +@Slf4j +public class SeleniumPoolServiceImpl implements ISeleniumPoolService, Runnable { + + /** + * 生成webdriver对象 + */ + private IWebDriverCreateService webDriverCreateService; + private SetupConfig setupConfig; + + private HashSet pool; + + /** + * 有效webdriver个数 + */ + @Getter + private int validNum = 0; + + public SeleniumPoolServiceImpl(IWebDriverCreateService webDriverCreateService, SetupConfig setupConfig) { + this.webDriverCreateService = webDriverCreateService; + this.setupConfig = setupConfig; + pool = new LinkedHashSet<>(setupConfig.getPoolSize()); + // 后台检测线程 + ThreadPoolExecutor monitor = new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new SynchronousQueue<>(), + ThreadFactoryBuilder.create().setNamePrefix("selenium-pool-monitor").build()); + monitor.execute(this); + } + + @Override + public synchronized IWebDriverService getWebDriver(long timeout) throws TimeoutException { + if (validNum <= 0 && pool.size() < setupConfig.getPoolSize()) { + pool.add(webDriverCreateService.createWebDriver(setupConfig)); + validNum ++; + } + + // 阻塞等待 + long startTime = System.currentTimeMillis(); + while (validNum <= 0) { + if (System.currentTimeMillis() - startTime >= timeout) { + throw new TimeoutException(); + } + try { + this.wait(timeout); + } catch (InterruptedException e) { + throw new TimeoutException(); + } + } + + // 获取 webdriver + for (IWebDriverService webDriver: pool) { + // 有效 且 不在使用 + if (webDriver.getStatus().isValid() && !webDriver.getStatus().isUsed()) { + validNum --; + webDriver.getStatus().startUse(); + return webDriver; + } + } + throw new NoWebDriverException(String.format("pool size %s, valid num %s, but no valid webdriver", pool.size(), validNum)); + } + + @Override + public synchronized void release(IWebDriverService webDriver) { + WebDriverStatus status = webDriver.getStatus(); + if (!status.isValid() || status.getFailedNum() > setupConfig.getMaxFailedNum()) { + log.info("{} invalid, remove", webDriver.getFlag()); + status.invalid(); + pool.remove(webDriver); + } else { + status.release(); + validNum ++; + this.notifyAll(); + } + } + + @Override + public void run() { + Random random = new Random(); + while (true) { + for (IWebDriverService webDriver: pool) { + WebDriverStatus status = webDriver.getStatus(); + + if (!status.isValid()) { + log.warn("{} checked invalid, go to remove", webDriver.getFlag()); + release(webDriver); + } + + if (status.getStartUseTime() - System.currentTimeMillis() > 300000) { + log.warn("{} used more than {}", webDriver.getFlag(), status.getStartUseTime() - System.currentTimeMillis()); + } + + // 运行一段时间后关闭， 防止浏览器内存问题， 30分钟上下5分钟，防止同时关闭/启动 + if (System.currentTimeMillis() - status.getStartUpTime() > random.nextInt(300000) + 1800000) { + webDriver.getStatus().invalid(); + } + + } + + ThreadUtil.sleep(20000); + } + } +} diff --git a/src/test/java/cn/heiliu/selenium/GetSingleWebDriver.java b/src/test/java/cn/heiliu/selenium/GetSingleWebDriver.java new file mode 100644 index 0000000..9e8efd6 --- /dev/null +++ b/src/test/java/cn/heiliu/selenium/GetSingleWebDriver.java @@ -0,0 +1,24 @@ +package cn.heiliu.selenium; + +import cn.heiliu.selenium.config.SetupConfig; +import cn.heiliu.selenium.service.IWebDriverCreateService; +import cn.heiliu.selenium.service.IWebDriverService; +import cn.heiliu.selenium.service.impl.FirefoxWebDriverCreateServiceImpl; + +/** + * @author wangyelou + * @date 2021/6/14 + */ +public class GetSingleWebDriver { + + public static void main(String[] args) { + SetupConfig setupConfig = SetupConfig.builder().binPath("C:\\Program Files\\Mozilla Firefox\\firefox.exe") + .enginePath("D:\\work\\firefox-selenium\\geckodriver.exe") + .proxyAuthExt("D:\\work\\firefox-selenium\\firefox_proxy_auth.xpi").build(); + + IWebDriverService webDriver = SeleniumManager.createFirefoxSelenium(setupConfig); + webDriver.fetch("http://www.baidu.com/"); + + } + +} diff --git a/src/test/java/cn/heiliu/selenium/GetWebDriverFromPool.java b/src/test/java/cn/heiliu/selenium/GetWebDriverFromPool.java new file mode 100644 index 0000000..4d63169 --- /dev/null +++ b/src/test/java/cn/heiliu/selenium/GetWebDriverFromPool.java @@ -0,0 +1,40 @@ +package cn.heiliu.selenium; + +import cn.heiliu.selenium.config.SetupConfig; +import cn.heiliu.selenium.service.ISeleniumPoolService; +import cn.heiliu.selenium.service.IWebDriverCreateService; +import cn.heiliu.selenium.service.IWebDriverService; +import cn.heiliu.selenium.service.impl.FirefoxWebDriverCreateServiceImpl; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * @author wangyelou + * @date 2021/6/14 + */ +public class GetWebDriverFromPool { + + public static void main(String[] args) { + + SetupConfig setupConfig = SetupConfig.builder().binPath("C:\\Program Files\\Mozilla Firefox\\firefox.exe") + .enginePath("D:\\work\\firefox-selenium\\geckodriver.exe") + .proxyAuthExt("D:\\work\\firefox-selenium\\firefox_proxy_auth.xpi") + .poolSize(3).build(); + ExecutorService executorService = Executors.newFixedThreadPool(3); + ISeleniumPoolService seleniumPool = SeleniumManager.createFirefoxSeleniumPool(setupConfig); + + for (int i=0; i<10; i++) { + executorService.execute(new Runnable() { + @Override + public void run() { + IWebDriverService webDriver = seleniumPool.getWebDriver(3000); + webDriver.fetch("http://www.badiu.com/"); + } + }); + } + + + } + +} diff --git a/target/classes/META-INF/SeleniumClient.kotlin_module b/target/classes/META-INF/SeleniumClient.kotlin_module new file mode 100644 index 0000000000000000000000000000000000000000..2983af70661ad375cc499ebc4da5a68ca46c532e GIT binary patch literal 16 RcmZQzU|?ooU|@t|egFVe02KfL literal 0 HcmV?d00001 diff --git a/target/classes/logback.xml b/target/classes/logback.xml new file mode 100644 index 0000000..490e425 --- /dev/null +++ b/target/classes/logback.xml @@ -0,0 +1,30 @@ + + + + + + + + %d %p (%file:%line\) - [%thread] [%X{key}] %m%n + UTF-8 + + + + + ${LOG_HOME}/server.log + + ${LOG_HOME}/server.%d.log + 7 + + + {"logtime": "%d", "level": "%level","thread": "%thread","class_name": "%class","line_number": "%line","message": "%message"}%n + UTF-8 + + + + + + + + + \ No newline at end of file -- Gitee From 1ce41481d528ee07b77daf05328f1aecf6407fb8 Mon Sep 17 00:00:00 2001 From: wangyelou Date: Tue, 15 Jun 2021 23:49:56 +0800 Subject: [PATCH 3/5] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=BB=93=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../heiliu/selenium/config/SetupConfig.java | 4 +- .../selenium/domain/WebDriverGetResult.java | 9 ++-- .../heiliu/selenium/enums/FileTypeEnum.java | 48 ------------------- .../impl/FirefoxWebDriverGetServiceImpl.java | 39 +++++++-------- .../impl/FirefoxWebDriverServiceImpl.java | 5 +- 5 files changed, 26 insertions(+), 79 deletions(-) delete mode 100644 src/main/java/cn/heiliu/selenium/enums/FileTypeEnum.java diff --git a/src/main/java/cn/heiliu/selenium/config/SetupConfig.java b/src/main/java/cn/heiliu/selenium/config/SetupConfig.java index 4e133fb..c7fa289 100644 --- a/src/main/java/cn/heiliu/selenium/config/SetupConfig.java +++ b/src/main/java/cn/heiliu/selenium/config/SetupConfig.java @@ -1,8 +1,10 @@ package cn.heiliu.selenium.config; +import cn.hutool.core.util.StrUtil; import lombok.Builder; import lombok.Getter; +import java.io.File; import java.util.UUID; /** @@ -67,7 +69,7 @@ public class SetupConfig { * @return */ public String getDownDir() { - return flag; + return StrUtil.hasEmpty(downDir) ? flag : downDir + File.separator + flag; } } diff --git a/src/main/java/cn/heiliu/selenium/domain/WebDriverGetResult.java b/src/main/java/cn/heiliu/selenium/domain/WebDriverGetResult.java index 16edcb7..7fc1bc3 100644 --- a/src/main/java/cn/heiliu/selenium/domain/WebDriverGetResult.java +++ b/src/main/java/cn/heiliu/selenium/domain/WebDriverGetResult.java @@ -1,6 +1,5 @@ package cn.heiliu.selenium.domain; -import cn.heiliu.selenium.enums.FileTypeEnum; import lombok.Getter; /** @@ -17,9 +16,9 @@ public class WebDriverGetResult { private byte[] content; /** - * 页面内容类型 + * 下载文件路径 */ - private FileTypeEnum contentType; + private String downFilePath; /** * 是否是下载文件 @@ -31,9 +30,9 @@ public class WebDriverGetResult { */ private long used; - public WebDriverGetResult(byte[] content, FileTypeEnum contentType, boolean isDownFile, long used) { + public WebDriverGetResult(byte[] content, String downFilePath, boolean isDownFile, long used) { this.content = content; - this.contentType = contentType; + this.downFilePath = downFilePath; this.isDownFile = isDownFile; this.used = used; } diff --git a/src/main/java/cn/heiliu/selenium/enums/FileTypeEnum.java b/src/main/java/cn/heiliu/selenium/enums/FileTypeEnum.java deleted file mode 100644 index c9fc39f..0000000 --- a/src/main/java/cn/heiliu/selenium/enums/FileTypeEnum.java +++ /dev/null @@ -1,48 +0,0 @@ -package cn.heiliu.selenium.enums; - -import lombok.Getter; - -/** - * @author wangyelou - * @date 2021/6/14 - */ -public enum FileTypeEnum { - - /** - * html - */ - HTML("html"), - - /** - * pdf - */ - PDF("pdf"), - - /** - * excel xls - */ - XLS("xls"), - - /** - * excel xlsx - */ - XLSX("xlsx"), - - /** - * zip - */ - ZIP("zip"), - - /** - * 未知类型 - */ - UNKNOWN("unknown"); - - @Getter - private String value; - - private FileTypeEnum(String value) { - this.value = value; - } - -} diff --git a/src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverGetServiceImpl.java b/src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverGetServiceImpl.java index 413c345..b6888c4 100644 --- a/src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverGetServiceImpl.java +++ b/src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverGetServiceImpl.java @@ -3,7 +3,6 @@ package cn.heiliu.selenium.service.impl; import cn.heiliu.selenium.domain.SeleniumProxy; import cn.heiliu.selenium.domain.SpecialUrl; import cn.heiliu.selenium.domain.WebDriverGetResult; -import cn.heiliu.selenium.enums.FileTypeEnum; import cn.heiliu.selenium.exception.DownFileTimeOutException; import cn.heiliu.selenium.service.IWebDriverGetService; import cn.hutool.core.io.FileUtil; @@ -15,8 +14,6 @@ import org.openqa.selenium.By; import org.openqa.selenium.NoSuchElementException; import java.io.File; -import java.util.HashMap; -import java.util.Map; /** * @author wangyelou @@ -36,14 +33,6 @@ public class FirefoxWebDriverGetServiceImpl implements IWebDriverGetService { private File downFile; private long startTime = System.currentTimeMillis(); - private static final Map EXT_RELATIONS = new HashMap<>(16); - - static { - EXT_RELATIONS.put("pdf", FileTypeEnum.PDF); - EXT_RELATIONS.put("zip", FileTypeEnum.ZIP); - EXT_RELATIONS.put("xls", FileTypeEnum.XLS); - EXT_RELATIONS.put("xlsx", FileTypeEnum.XLSX); - } public FirefoxWebDriverGetServiceImpl() { } @@ -94,21 +83,15 @@ public class FirefoxWebDriverGetServiceImpl implements IWebDriverGetService { * 下载结果处理 */ private void dealResult() { - long startTime = System.currentTimeMillis(); + long startCheckTime = System.currentTimeMillis(); boolean htmlFlag = false; boolean downFlag = false; - while (!htmlFlag && !downFlag && System.currentTimeMillis() - startTime < timeout) { - // html页面元素检查 - if (!StrUtil.hasEmpty(waitXpath) && !webDriver.findElements(By.xpath(waitXpath)).isEmpty()) { - log.info("find element with xpath: {}", waitXpath); - htmlFlag = true; - } - + while (!htmlFlag && !downFlag && System.currentTimeMillis() - startCheckTime < timeout) { // 下载文件检查 if (FileUtil.exist(downDir)) { log.info("file downloading..."); // 检测到文件下载，则等待下载完成 - while (FileUtil.ls(downDir).length > 1 && System.currentTimeMillis() - startTime < timeout) { + while (FileUtil.ls(downDir).length > 1 && System.currentTimeMillis() - startCheckTime < timeout) { ThreadUtil.sleep(1000); } @@ -120,10 +103,22 @@ public class FirefoxWebDriverGetServiceImpl implements IWebDriverGetService { downFlag = true; } + + // html页面元素检查 + if (!StrUtil.hasEmpty(waitXpath)) { + if (!webDriver.findElements(By.xpath(waitXpath)).isEmpty()) { + log.info("find element with xpath: {}", waitXpath); + htmlFlag = true; + } + } else { + // 如果没有设置wait xpath 则检查完是否有下载文件后就退出 + break; + } + ThreadUtil.sleep(500); } - if (System.currentTimeMillis() - startTime > timeout) { + if (System.currentTimeMillis() - startCheckTime > timeout) { if (downFlag) { throw new DownFileTimeOutException("web driver download file time out " + timeout); } else { @@ -142,7 +137,7 @@ public class FirefoxWebDriverGetServiceImpl implements IWebDriverGetService { public WebDriverGetResult getResult() { return new WebDriverGetResult( FileUtil.isFile(downFile) ? FileUtil.readBytes(downFile) : webDriver.getPageSource().getBytes(), - FileUtil.isFile(downFile) ? EXT_RELATIONS.getOrDefault(FileUtil.extName(downFile), FileTypeEnum.UNKNOWN) : FileTypeEnum.HTML, + downFile != null ? downFile.getAbsolutePath() : null, downFile != null, System.currentTimeMillis() - startTime ); diff --git a/src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverServiceImpl.java b/src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverServiceImpl.java index 4236ca6..92be8da 100644 --- a/src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverServiceImpl.java +++ b/src/main/java/cn/heiliu/selenium/service/impl/FirefoxWebDriverServiceImpl.java @@ -21,7 +21,6 @@ public class FirefoxWebDriverServiceImpl extends FirefoxDriver implements IWebDr private static final String INSTALL_EXTENSION = "installExtension"; private static final long DEFAULT_TIME_OUT = 10000; - private static final String DEFAULT_WAIT_XPATH = "//div | //p | //table | //tr | //td | //span"; private static final String FIREFOX_SPECIAL_URL_PREFIX = "about:"; private static AtomicInteger num = new AtomicInteger(1); @@ -88,7 +87,7 @@ public class FirefoxWebDriverServiceImpl extends FirefoxDriver implements IWebDr @Override public IWebDriverGetService fetch(String url, SeleniumProxy proxy) { - return fetch(url, proxy, DEFAULT_WAIT_XPATH, DEFAULT_TIME_OUT); + return fetch(url, proxy, null, DEFAULT_TIME_OUT); } @Override @@ -106,7 +105,7 @@ public class FirefoxWebDriverServiceImpl extends FirefoxDriver implements IWebDr super.get(url); return new FirefoxWebDriverGetServiceImpl(); } else { - return fetch(url, null, DEFAULT_WAIT_XPATH, DEFAULT_TIME_OUT); + return fetch(url, null, null, DEFAULT_TIME_OUT); } } -- Gitee From 93507560d23160cf340dcfe2aaa8a5ad2bb8c0f9 Mon Sep 17 00:00:00 2001 From: wangyelou Date: Wed, 16 Jun 2021 01:02:25 +0800 Subject: [PATCH 4/5] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=BB=93=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 35 ++++++++---------- src/main/resources/logback.xml | 30 --------------- .../META-INF/SeleniumClient.kotlin_module | Bin 16 -> 0 bytes target/maven-archiver/pom.properties | 5 +++ .../compile/default-compile/createdFiles.lst | 18 +++++++++ .../compile/default-compile/inputFiles.lst | 17 +++++++++ .../default-testCompile/createdFiles.lst | 3 ++ .../default-testCompile/inputFiles.lst | 2 + 8 files changed, 61 insertions(+), 49 deletions(-) delete mode 100644 src/main/resources/logback.xml delete mode 100644 target/classes/META-INF/SeleniumClient.kotlin_module create mode 100644 target/maven-archiver/pom.properties create mode 100644 target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst create mode 100644 target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst create mode 100644 target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/createdFiles.lst create mode 100644 target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/inputFiles.lst diff --git a/pom.xml b/pom.xml index 302180d..13c1223 100644 --- a/pom.xml +++ b/pom.xml @@ -9,11 +9,11 @@ 1.0-SNAPSHOT - 11 - true + 1.8 + ${java.version} + ${java.version} - @@ -40,22 +40,19 @@ 1.8.0-beta2 - - ch.qos.logback - logback-core - 1.2.3 - + - - ch.qos.logback - logback-classic - 1.2.3 - + + + + + maven-compiler-plugin + + true + + + + + - - 　　org.slf4j - 　　slf4j-simple - 1.8.0-beta2 - - \ No newline at end of file diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml deleted file mode 100644 index 490e425..0000000 --- a/src/main/resources/logback.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - - %d %p (%file:%line\) - [%thread] [%X{key}] %m%n - UTF-8 - - - - - ${LOG_HOME}/server.log - - ${LOG_HOME}/server.%d.log - 7 - - - {"logtime": "%d", "level": "%level","thread": "%thread","class_name": "%class","line_number": "%line","message": "%message"}%n - UTF-8 - - - - - - - - - \ No newline at end of file diff --git a/target/classes/META-INF/SeleniumClient.kotlin_module b/target/classes/META-INF/SeleniumClient.kotlin_module deleted file mode 100644 index 2983af70661ad375cc499ebc4da5a68ca46c532e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 RcmZQzU|?ooU|@t|egFVe02KfL diff --git a/target/maven-archiver/pom.properties b/target/maven-archiver/pom.properties new file mode 100644 index 0000000..d051b0f --- /dev/null +++ b/target/maven-archiver/pom.properties @@ -0,0 +1,5 @@ +#Generated by Maven +#Wed Jun 16 01:00:05 CST 2021 +groupId=cn.heiliu.selenium +artifactId=selenium-client +version=1.0-SNAPSHOT diff --git a/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst b/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst new file mode 100644 index 0000000..8e1c30e --- /dev/null +++ b/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst @@ -0,0 +1,18 @@ +cn\heiliu\selenium\exception\DownFileTimeOutException.class +cn\heiliu\selenium\service\impl\SeleniumPoolServiceImpl.class +cn\heiliu\selenium\service\ISeleniumPoolService.class +cn\heiliu\selenium\domain\WebDriverStatus.class +cn\heiliu\selenium\exception\UnsupportedMethodException.class +cn\heiliu\selenium\domain\SeleniumProxy.class +cn\heiliu\selenium\service\impl\FirefoxWebDriverCreateServiceImpl.class +cn\heiliu\selenium\service\IWebDriverCreateService.class +cn\heiliu\selenium\service\impl\FirefoxWebDriverServiceImpl.class +cn\heiliu\selenium\domain\SpecialUrl.class +cn\heiliu\selenium\config\SetupConfig$SetupConfigBuilder.class +cn\heiliu\selenium\exception\NoWebDriverException.class +cn\heiliu\selenium\config\SetupConfig.class +cn\heiliu\selenium\service\impl\FirefoxWebDriverGetServiceImpl.class +cn\heiliu\selenium\service\IWebDriverGetService.class +cn\heiliu\selenium\service\IWebDriverService.class +cn\heiliu\selenium\SeleniumManager.class +cn\heiliu\selenium\domain\WebDriverGetResult.class diff --git a/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst b/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst new file mode 100644 index 0000000..514a448 --- /dev/null +++ b/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst @@ -0,0 +1,17 @@ +D:\work\SeleniumClient\src\main\java\cn\heiliu\selenium\exception\NoWebDriverException.java +D:\work\SeleniumClient\src\main\java\cn\heiliu\selenium\config\SetupConfig.java +D:\work\SeleniumClient\src\main\java\cn\heiliu\selenium\SeleniumManager.java +D:\work\SeleniumClient\src\main\java\cn\heiliu\selenium\domain\SeleniumProxy.java +D:\work\SeleniumClient\src\main\java\cn\heiliu\selenium\service\impl\FirefoxWebDriverServiceImpl.java +D:\work\SeleniumClient\src\main\java\cn\heiliu\selenium\exception\DownFileTimeOutException.java +D:\work\SeleniumClient\src\main\java\cn\heiliu\selenium\domain\WebDriverGetResult.java +D:\work\SeleniumClient\src\main\java\cn\heiliu\selenium\service\IWebDriverService.java +D:\work\SeleniumClient\src\main\java\cn\heiliu\selenium\domain\SpecialUrl.java +D:\work\SeleniumClient\src\main\java\cn\heiliu\selenium\service\IWebDriverGetService.java +D:\work\SeleniumClient\src\main\java\cn\heiliu\selenium\service\impl\FirefoxWebDriverGetServiceImpl.java +D:\work\SeleniumClient\src\main\java\cn\heiliu\selenium\service\impl\SeleniumPoolServiceImpl.java +D:\work\SeleniumClient\src\main\java\cn\heiliu\selenium\exception\UnsupportedMethodException.java +D:\work\SeleniumClient\src\main\java\cn\heiliu\selenium\service\ISeleniumPoolService.java +D:\work\SeleniumClient\src\main\java\cn\heiliu\selenium\domain\WebDriverStatus.java +D:\work\SeleniumClient\src\main\java\cn\heiliu\selenium\service\impl\FirefoxWebDriverCreateServiceImpl.java +D:\work\SeleniumClient\src\main\java\cn\heiliu\selenium\service\IWebDriverCreateService.java diff --git a/target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/createdFiles.lst b/target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/createdFiles.lst new file mode 100644 index 0000000..d1fde3c --- /dev/null +++ b/target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/createdFiles.lst @@ -0,0 +1,3 @@ +cn\heiliu\selenium\GetSingleWebDriver.class +cn\heiliu\selenium\GetWebDriverFromPool$1.class +cn\heiliu\selenium\GetWebDriverFromPool.class diff --git a/target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/inputFiles.lst b/target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/inputFiles.lst new file mode 100644 index 0000000..1363f9a --- /dev/null +++ b/target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/inputFiles.lst @@ -0,0 +1,2 @@ +D:\work\SeleniumClient\src\test\java\cn\heiliu\selenium\GetWebDriverFromPool.java +D:\work\SeleniumClient\src\test\java\cn\heiliu\selenium\GetSingleWebDriver.java -- Gitee From 95518c9befffe47a4f3d51f43eff56e93e2f3f91 Mon Sep 17 00:00:00 2001 From: wangyelou Date: Wed, 16 Jun 2021 01:13:14 +0800 Subject: [PATCH 5/5] readme --- README.md | 37 +++++++++++++++++++ .../heiliu/selenium/GetWebDriverFromPool.java | 6 ++- 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..506b8af --- /dev/null +++ b/README.md @@ -0,0 +1,37 @@ +#### 引入 +```$xslt + + cn.heiliu.selenium + selenium-client + 1.0.0 + +``` + +#### 使用 + +* 单独使用 +```java +SetupConfig setupConfig = SetupConfig.builder().binPath("C:\\Program Files\\Mozilla Firefox\\firefox.exe") + .enginePath("D:\\work\\firefox-selenium\\geckodriver.exe") + .proxyAuthExt("D:\\work\\firefox-selenium\\firefox_proxy_auth.xpi").build(); + +IWebDriverService webDriver = SeleniumManager.createFirefoxSelenium(setupConfig); +webDriver.fetch("https://www.baidu.com/"); +``` + +* 连接池使用 +```java +SetupConfig setupConfig = SetupConfig.builder().binPath("C:\\Program Files\\Mozilla Firefox\\firefox.exe") + .enginePath("D:\\work\\firefox-selenium\\geckodriver.exe") + .proxyAuthExt("D:\\work\\firefox-selenium\\firefox_proxy_auth.xpi") + .poolSize(3).build(); +ExecutorService executorService = Executors.newFixedThreadPool(3); +ISeleniumPoolService seleniumPool = SeleniumManager.createFirefoxSeleniumPool(setupConfig); + +IWebDriverService webDriver = seleniumPool.getWebDriver(3000); +try { + webDriver.fetch("http://www.badiu.com/"); +} finally { + seleniumPool.release(webDriver); +} +``` \ No newline at end of file diff --git a/src/test/java/cn/heiliu/selenium/GetWebDriverFromPool.java b/src/test/java/cn/heiliu/selenium/GetWebDriverFromPool.java index 4d63169..caa6c77 100644 --- a/src/test/java/cn/heiliu/selenium/GetWebDriverFromPool.java +++ b/src/test/java/cn/heiliu/selenium/GetWebDriverFromPool.java @@ -29,7 +29,11 @@ public class GetWebDriverFromPool { @Override public void run() { IWebDriverService webDriver = seleniumPool.getWebDriver(3000); - webDriver.fetch("http://www.badiu.com/"); + try { + webDriver.fetch("http://www.badiu.com/"); + } finally { + seleniumPool.release(webDriver); + } } }); } -- Gitee