diff --git a/application/admin/library/traits/Backend.php b/application/admin/library/traits/Backend.php
index fc7032b509ab0e943097ca99ed700c9d66d5561a..d0aa3f9ac0b3f9cb2cb2b67230cdf57e9e3a3a89 100755
--- a/application/admin/library/traits/Backend.php
+++ b/application/admin/library/traits/Backend.php
@@ -4,11 +4,12 @@ namespace app\admin\library\traits;
use app\admin\library\Auth;
use Exception;
+use PhpOffice\PhpSpreadsheet\IOFactory;
+use PhpOffice\PhpSpreadsheet\Spreadsheet;
+use PhpOffice\PhpSpreadsheet\Reader\IReadFilter;
use PhpOffice\PhpSpreadsheet\Cell\Coordinate;
-use PhpOffice\PhpSpreadsheet\Reader\Xlsx;
-use PhpOffice\PhpSpreadsheet\Reader\Xls;
-use PhpOffice\PhpSpreadsheet\Reader\Csv;
use think\Db;
+use think\Hook;
use think\db\exception\BindParamException;
use think\db\exception\DataNotFoundException;
use think\db\exception\ModelNotFoundException;
@@ -350,132 +351,283 @@ trait Backend
*/
protected function import()
{
+
+ // 后台运行
+ ignore_user_abort(true);
+
+ // 取消超时
+ set_time_limit(0);
+
+ $timer1 = microtime(true);
+ $usage1 = memory_get_usage();
+
+ //取得上传文件
+
$file = $this->request->request('file');
if (!$file) {
$this->error(__('Parameter %s can not be empty', 'file'));
}
- $filePath = ROOT_PATH . DS . 'public' . DS . $file;
+
+ $filePath = ROOT_PATH.DS.'public'.DS.$file;
if (!is_file($filePath)) {
$this->error(__('No results were found'));
}
- //实例化reader
- $ext = pathinfo($filePath, PATHINFO_EXTENSION);
- if (!in_array($ext, ['csv', 'xls', 'xlsx'])) {
- $this->error(__('Unknown data format'));
- }
- if ($ext === 'csv') {
- $file = fopen($filePath, 'r');
- $filePath = tempnam(sys_get_temp_dir(), 'import_csv');
- $fp = fopen($filePath, 'w');
- $n = 0;
- while ($line = fgets($file)) {
- $line = rtrim($line, "\n\r\0");
- $encoding = mb_detect_encoding($line, ['utf-8', 'gbk', 'latin1', 'big5']);
- if ($encoding !== 'utf-8') {
- $line = mb_convert_encoding($line, 'utf-8', $encoding);
- }
- if ($n == 0 || preg_match('/^".*"$/', $line)) {
- fwrite($fp, $line . "\n");
- } else {
- fwrite($fp, '"' . str_replace(['"', ','], ['""', '","'], $line) . "\"\n");
- }
- $n++;
- }
- fclose($file) || fclose($fp);
- $reader = new Csv();
- } elseif ($ext === 'xls') {
- $reader = new Xls();
- } else {
- $reader = new Xlsx();
+ $insert = [];
+ $encode = 'utf-8';
+ $charSet = setlocale(LC_CTYPE, 0);
+ $fileType = strtolower(IOFactory::identify($filePath));
+
+ // 检查文件类型
+
+ if (!in_array($fileType, ['csv', 'xls', 'xlsx'])) {
+ $this->error(__('Unknown data format'));
}
//导入文件首行类型,默认是注释,如果需要使用字段名称请使用name
$importHeadType = isset($this->importHeadType) ? $this->importHeadType : 'comment';
- $table = $this->model->getQuery()->getTable();
+ $model = get_class($this->model);
+ $table = $this->model->getQuery()->getTable();
$database = \think\Config::get('database.database');
$fieldArr = [];
- $list = db()->query("SELECT COLUMN_NAME,COLUMN_COMMENT FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = ? AND TABLE_SCHEMA = ?", [$table, $database]);
+ $list = db()->query("SELECT COLUMN_NAME,COLUMN_COMMENT FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = ? AND TABLE_SCHEMA = ?", [$table, $database]);
foreach ($list as $k => $v) {
- if ($importHeadType == 'comment') {
- $v['COLUMN_COMMENT'] = explode(':', $v['COLUMN_COMMENT'])[0]; //字段备注有:时截取
+ if ('comment' == $importHeadType) {
+ $v['COLUMN_COMMENT'] = explode(':', $v['COLUMN_COMMENT'])[0]; //字段备注有:时截取
$fieldArr[$v['COLUMN_COMMENT']] = $v['COLUMN_NAME'];
} else {
$fieldArr[$v['COLUMN_NAME']] = $v['COLUMN_NAME'];
}
}
+ // 取得csv文件编码
+
+ if ('csv' === $fileType) {
+
+ // 读取文件的前 1KB 进行检测
+
+ $handle = fopen($filePath, 'r');
+
+ rewind($handle);
+ $snippet = fread($handle, 1024);
+ fclose($handle);
+
+ $encodes = ['utf-8', 'gbk', 'gb18030', 'latin1', 'big5'];
+ $detected = mb_detect_encoding($snippet, $encodes);
+
+ if (false !== $detected) {
+ $encode = $detected;
+ }
+ }
+
//加载文件
- $insert = [];
+
try {
- if (!$PHPExcel = $reader->load($filePath)) {
- $this->error(__('Unknown data format'));
+
+ // windows系统php7环境csv编码解析问题
+
+ if ('C' !== $charSet) {
+ setlocale(LC_CTYPE, 'C');
}
- $currentSheet = $PHPExcel->getSheet(0); //读取文件中的第一个工作表
- $allColumn = $currentSheet->getHighestDataColumn(); //取得最大的列号
- $allRow = $currentSheet->getHighestRow(); //取得一共有多少行
- $maxColumnNumber = Coordinate::columnIndexFromString($allColumn);
- $fields = [];
- for ($currentRow = 1; $currentRow <= 1; $currentRow++) {
- for ($currentColumn = 1; $currentColumn <= $maxColumnNumber; $currentColumn++) {
- $val = $currentSheet->getCellByColumnAndRow($currentColumn, $currentRow)->getValue();
- $fields[] = $val;
- }
+
+ // 加载表格
+
+ $fileReader = IOFactory::createReaderForFile($filePath);
+ $fileReader->setReadDataOnly(true);
+ $sheetlist = $fileReader->listWorksheetInfo($filePath);
+ $sheetInfo = isset($sheetlist[0]) ? (object) $sheetlist[0] : null;
+
+ if (empty($sheetInfo)) {
+ throw new Exception('读取工作表信息失败');
+ }
+
+ if ('csv' === $fileType) {
+ $fileReader->setInputEncoding($encode);
}
- for ($currentRow = 2; $currentRow <= $allRow; $currentRow++) {
- $values = [];
- for ($currentColumn = 1; $currentColumn <= $maxColumnNumber; $currentColumn++) {
- $val = $currentSheet->getCellByColumnAndRow($currentColumn, $currentRow)->getValue();
- $values[] = is_null($val) ? '' : $val;
+ // 分块读取配置
+
+ $header = [];
+ $chunkSize = 10000;
+ $chunkSheet = new Spreadsheet();
+ $chunkFilter = new IReadFilter\ChunkReadFilter();
+ $chunkReader = IOFactory::createReaderForFile($filePath);
+ $chunkReader->setReadDataOnly(true);
+ $chunkReader->setReadFilter($chunkFilter);
+
+ if ('csv' === $fileType) {
+ $chunkReader->setContiguous(true);
+ $chunkReader->setInputEncoding($encode);
+ }
+
+ // 分块读取数据
+
+ for ($currentRow = 1; $currentRow <= $sheetInfo->totalRows; $currentRow += $chunkSize) {
+
+ $chunkFilter->setChunk($currentRow, $chunkSize);
+ $chunkReader->setLoadSheetsOnly($sheetInfo->worksheetName);
+
+ if ('csv' === $fileType) {
+ $chunkReader->setSheetIndex(0);
+ $chunkSheet = $chunkReader->loadIntoExisting($filePath, $chunkSheet);
+ } else {
+ $chunkSheet = $chunkReader->load($filePath);
}
- $row = [];
- $temp = array_combine($fields, $values);
- foreach ($temp as $k => $v) {
- if (isset($fieldArr[$k]) && $k !== '') {
- $row[$fieldArr[$k]] = $v;
+
+ $activeSheet = $chunkSheet->getSheet(0);
+
+ // 取得当前工作表名称
+ $worksheetName = $activeSheet->getTitle();
+ // 取得最大的列号(ABC..)
+ $lastColumnLetter = $activeSheet->getHighestDataColumn();
+ // 取得最大的列索引(123..)
+ $lastColumnIndex = Coordinate::columnIndexFromString($lastColumnLetter);
+ // 取得总行数
+ $totalRows = $activeSheet->getHighestRow();
+ // 取得总列数
+ $totalColumns = $lastColumnIndex + 1;
+
+ // 转为数组
+ $sheetRange = sprintf('A1:%s%d', $lastColumnLetter, $totalRows);
+ $sheetArray = $activeSheet->rangeToArray($sheetRange, null, false, false, false);
+
+ // 数据处理
+ if (!empty($sheetArray)) {
+
+ $startIndex = 1;
+ $currentIndex = 1;
+
+ if (1 == $currentRow) {
+ $startIndex++;
+ $header = $sheetArray[0];
+ }
+
+ foreach ($sheetArray as $row) {
+ if ($currentIndex >= $startIndex) {
+ foreach ($row as &$cell) {
+ $cell = is_null($cell) ? '' : $cell;
+ }
+ $vals = [];
+ $temp = array_combine($header, $row);
+ foreach ($temp as $k => $v) {
+ if (isset($fieldArr[$k]) && '' !== $k) {
+ $vals[$fieldArr[$k]] = $v;
+ }
+ }
+ $insert[] = $vals;
+ }
+ $currentIndex++;
}
}
- if ($row) {
- $insert[] = $row;
- }
+
+ $chunkSheet->removeSheetByIndex(0);
+ $chunkSheet->createSheet(0);
+
}
+
} catch (Exception $exception) {
- $this->error($exception->getMessage());
+ $msg = $exception->getMessage();
+ if (preg_match("/.*The filename (.+) is not recognised as an OLE file.*/is", $msg, $matches)) {
+ $msg = "读取文件失败, 建议重新将文件另存为csv、xls或xslx格式";
+ };
+ $this->error($msg);
}
+
if (!$insert) {
$this->error(__('No rows were updated'));
}
- try {
- //是否包含admin_id字段
- $has_admin_id = false;
- foreach ($fieldArr as $name => $key) {
- if ($key == 'admin_id') {
- $has_admin_id = true;
- break;
+ //是否包含admin_id字段
+
+ $has_admin_id = false;
+ foreach ($fieldArr as $name => $key) {
+ if ('admin_id' == $key) {
+ $has_admin_id = true;
+ break;
+ }
+ }
+
+ if ($has_admin_id) {
+ $auth = Auth::instance();
+ foreach ($insert as &$val) {
+ if (!isset($val['admin_id']) || empty($val['admin_id'])) {
+ $val['admin_id'] = $auth->isLogin() ? $auth->id : 0;
}
}
- if ($has_admin_id) {
- $auth = Auth::instance();
- foreach ($insert as &$val) {
- if (!isset($val['admin_id']) || empty($val['admin_id'])) {
- $val['admin_id'] = $auth->isLogin() ? $auth->id : 0;
- }
+ }
+
+ // 批量插入数据
+
+ $affectRows = 0;
+ $repeatRows = 0;
+
+ foreach ($insert as &$data) {
+
+ $errMsg = null;
+ $error = null;
+ $event = null;
+
+ $hook = [
+ 'event' => 'before_import',
+ 'class' => static::class,
+ 'model' => $model,
+ 'data' => $data,
+ 'error' => null,
+ ];
+
+ Hook::listen('table_import', $hook);
+
+ try {
+
+ $affectRows += (new $model($hook['data']))->save();
+ $event = 'after_import';
+
+ } catch (PDOException $e) {
+
+ $error = $e;
+ $event = 'pdo_exception';
+
+ if (preg_match("/.+Integrity constraint violation: 1062 Duplicate entry '(.+)' for key '(.+)'/is", $error, $matches)) {
+ $errMsg = "导入失败，包含【{$matches[1]}】的记录已存在";
+ $event = 'pdo_duplicate_entry';
+ $repeatRows++;
}
+
+ } catch (Exception $e) {
+ $error = $e;
+ $event = 'exception';
}
- $this->model->saveAll($insert);
- } catch (PDOException $exception) {
- $msg = $exception->getMessage();
- if (preg_match("/.+Integrity constraint violation: 1062 Duplicate entry '(.+)' for key '(.+)'/is", $msg, $matches)) {
- $msg = "导入失败，包含【{$matches[1]}】的记录已存在";
- };
- $this->error($msg);
- } catch (Exception $e) {
- $this->error($e->getMessage());
+
+ $hook['event'] = $event;
+ $hook['error'] = $error;
+
+ Hook::listen('table_import', $hook);
+ $error = $hook['error'];
+
+ if ($error) {
+ $errMsg = $errMsg ?: $error->getMessage();
+ $this->error($errMsg);
+ }
+
}
- $this->success();
+ if (!$affectRows) {
+ $this->error(__('No rows were updated'));
+ }
+
+ $timer2 = microtime(true);
+ $usage2 = memory_get_usage();
+ $usage = round(($usage2 - $usage1) / 1024 / 1024, 2);
+ $timer = round($timer2 - $timer1, 3);
+
+ $success = vsprintf("导入成功!
%s
%s
%s
%s", [
+ sprintf("新增数据: %d 条", $affectRows),
+ sprintf("重复数据: %d 条", $repeatRows),
+ sprintf("内存占用: %s MB", $usage),
+ sprintf("总共耗时: %s 秒", $timer),
+ ]);
+
+ $this->success($success);
}
}