优化增强导入功能代码, 增加Hook(新增数据, 重复数据, 错误数据)

application/admin/library/traits/Backend.php

@@ -4,11 +4,12 @@ namespace app\admin\library\traits;
 
 use app\admin\library\Auth;
 use Exception;
+use PhpOffice\PhpSpreadsheet\IOFactory;
+use PhpOffice\PhpSpreadsheet\Spreadsheet;
+use PhpOffice\PhpSpreadsheet\Reader\IReadFilter;
 use PhpOffice\PhpSpreadsheet\Cell\Coordinate;
-use PhpOffice\PhpSpreadsheet\Reader\Xlsx;
-use PhpOffice\PhpSpreadsheet\Reader\Xls;
-use PhpOffice\PhpSpreadsheet\Reader\Csv;
 use think\Db;
+use think\Hook;
 use think\db\exception\BindParamException;
 use think\db\exception\DataNotFoundException;
 use think\db\exception\ModelNotFoundException;
@@ -350,132 +351,283 @@ trait Backend
      */
     protected function import()
     {
+
+        // 后台运行
+        ignore_user_abort(true);
+
+        // 取消超时
+        set_time_limit(0);
+
+        $timer1 = microtime(true);
+        $usage1 = memory_get_usage();
+
+        //取得上传文件
+
         $file = $this->request->request('file');
         if (!$file) {
             $this->error(__('Parameter %s can not be empty', 'file'));
         }
-        $filePath = ROOT_PATH . DS . 'public' . DS . $file;
+
+        $filePath = ROOT_PATH.DS.'public'.DS.$file;
         if (!is_file($filePath)) {
             $this->error(__('No results were found'));
         }
-        //实例化reader
-        $ext = pathinfo($filePath, PATHINFO_EXTENSION);
-        if (!in_array($ext, ['csv', 'xls', 'xlsx'])) {
-            $this->error(__('Unknown data format'));
-        }
-        if ($ext === 'csv') {
-            $file = fopen($filePath, 'r');
-            $filePath = tempnam(sys_get_temp_dir(), 'import_csv');
-            $fp = fopen($filePath, 'w');
-            $n = 0;
-            while ($line = fgets($file)) {
-                $line = rtrim($line, "\n\r\0");
-                $encoding = mb_detect_encoding($line, ['utf-8', 'gbk', 'latin1', 'big5']);
-                if ($encoding !== 'utf-8') {
-                    $line = mb_convert_encoding($line, 'utf-8', $encoding);
-                }
-                if ($n == 0 || preg_match('/^".*"$/', $line)) {
-                    fwrite($fp, $line . "\n");
-                } else {
-                    fwrite($fp, '"' . str_replace(['"', ','], ['""', '","'], $line) . "\"\n");
-                }
-                $n++;
-            }
-            fclose($file) || fclose($fp);
 
-            $reader = new Csv();
-        } elseif ($ext === 'xls') {
-            $reader = new Xls();
-        } else {
-            $reader = new Xlsx();
+        $insert   = [];
+        $encode   = 'utf-8';
+        $charSet  = setlocale(LC_CTYPE, 0);
+        $fileType = strtolower(IOFactory::identify($filePath));
+
+        // 检查文件类型
+
+        if (!in_array($fileType, ['csv', 'xls', 'xlsx'])) {
+            $this->error(__('Unknown data format'));
         }
 
         //导入文件首行类型,默认是注释,如果需要使用字段名称请使用name
         $importHeadType = isset($this->importHeadType) ? $this->importHeadType : 'comment';
 
-        $table = $this->model->getQuery()->getTable();
+        $model    = get_class($this->model);
+        $table    = $this->model->getQuery()->getTable();
         $database = \think\Config::get('database.database');
         $fieldArr = [];
-        $list = db()->query("SELECT COLUMN_NAME,COLUMN_COMMENT FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = ? AND TABLE_SCHEMA = ?", [$table, $database]);
+        $list     = db()->query("SELECT COLUMN_NAME,COLUMN_COMMENT FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = ? AND TABLE_SCHEMA = ?", [$table, $database]);
         foreach ($list as $k => $v) {
-            if ($importHeadType == 'comment') {
-                $v['COLUMN_COMMENT'] = explode(':', $v['COLUMN_COMMENT'])[0]; //字段备注有:时截取
+            if ('comment' == $importHeadType) {
+                $v['COLUMN_COMMENT']            = explode(':', $v['COLUMN_COMMENT'])[0]; //字段备注有:时截取
                 $fieldArr[$v['COLUMN_COMMENT']] = $v['COLUMN_NAME'];
             } else {
                 $fieldArr[$v['COLUMN_NAME']] = $v['COLUMN_NAME'];
             }
         }
 
-        //加载文件
-        $insert = [];
-        try {
-            if (!$PHPExcel = $reader->load($filePath)) {
-                $this->error(__('Unknown data format'));
+        // 取得csv文件编码
+
+        if ('csv' === $fileType) {
+
+            // 读取文件的前 1KB 进行检测
+
+            $handle = fopen($filePath, 'r');
+
+            rewind($handle);
+            $snippet = fread($handle, 1024);
+            fclose($handle);
+
+            $encodes  = ['utf-8', 'gbk', 'gb18030', 'latin1', 'big5'];
+            $detected = mb_detect_encoding($snippet, $encodes);
+
+            if (false !== $detected) {
+                $encode = $detected;
             }
-            $currentSheet = $PHPExcel->getSheet(0);  //读取文件中的第一个工作表
-            $allColumn = $currentSheet->getHighestDataColumn(); //取得最大的列号
-            $allRow = $currentSheet->getHighestRow(); //取得一共有多少行
-            $maxColumnNumber = Coordinate::columnIndexFromString($allColumn);
-            $fields = [];
-            for ($currentRow = 1; $currentRow <= 1; $currentRow++) {
-                for ($currentColumn = 1; $currentColumn <= $maxColumnNumber; $currentColumn++) {
-                    $val = $currentSheet->getCellByColumnAndRow($currentColumn, $currentRow)->getValue();
-                    $fields[] = $val;
-                }
+        }
+
+        //加载文件
+
+        try {
+
+            // windows系统php7环境csv编码解析问题
+
+            if ('C' !== $charSet) {
+                setlocale(LC_CTYPE, 'C');
             }
 
-            for ($currentRow = 2; $currentRow <= $allRow; $currentRow++) {
-                $values = [];
-                for ($currentColumn = 1; $currentColumn <= $maxColumnNumber; $currentColumn++) {
-                    $val = $currentSheet->getCellByColumnAndRow($currentColumn, $currentRow)->getValue();
-                    $values[] = is_null($val) ? '' : $val;
+            // 加载表格
+
+            $fileReader = IOFactory::createReaderForFile($filePath);
+            $fileReader->setReadDataOnly(true);
+            $sheetlist = $fileReader->listWorksheetInfo($filePath);
+            $sheetInfo = isset($sheetlist[0]) ? (object) $sheetlist[0] : null;
+
+            if (empty($sheetInfo)) {
+                throw new Exception('读取工作表信息失败');
+            }
+
+            if ('csv' === $fileType) {
+                $fileReader->setInputEncoding($encode);
+            }
+
+            // 分块读取配置
+
+            $header      = [];
+            $chunkSize   = 10000;
+            $chunkSheet  = new Spreadsheet();
+            $chunkFilter = new IReadFilter\ChunkReadFilter();
+            $chunkReader = IOFactory::createReaderForFile($filePath);
+            $chunkReader->setReadDataOnly(true);
+            $chunkReader->setReadFilter($chunkFilter);
+
+            if ('csv' === $fileType) {
+                $chunkReader->setContiguous(true);
+                $chunkReader->setInputEncoding($encode);
+            }
+
+            // 分块读取数据
+
+            for ($currentRow = 1; $currentRow <= $sheetInfo->totalRows; $currentRow += $chunkSize) {
+
+                $chunkFilter->setChunk($currentRow, $chunkSize);
+                $chunkReader->setLoadSheetsOnly($sheetInfo->worksheetName);
+
+                if ('csv' === $fileType) {
+                    $chunkReader->setSheetIndex(0);
+                    $chunkSheet = $chunkReader->loadIntoExisting($filePath, $chunkSheet);
+                } else {
+                    $chunkSheet = $chunkReader->load($filePath);
                 }
-                $row = [];
-                $temp = array_combine($fields, $values);
-                foreach ($temp as $k => $v) {
-                    if (isset($fieldArr[$k]) && $k !== '') {
-                        $row[$fieldArr[$k]] = $v;
+
+                $activeSheet = $chunkSheet->getSheet(0);
+
+                // 取得当前工作表名称
+                $worksheetName = $activeSheet->getTitle();
+                // 取得最大的列号(ABC..)
+                $lastColumnLetter = $activeSheet->getHighestDataColumn();
+                // 取得最大的列索引(123..)
+                $lastColumnIndex = Coordinate::columnIndexFromString($lastColumnLetter);
+                // 取得总行数
+                $totalRows = $activeSheet->getHighestRow();
+                // 取得总列数
+                $totalColumns = $lastColumnIndex + 1;
+
+                // 转为数组
+                $sheetRange = sprintf('A1:%s%d', $lastColumnLetter, $totalRows);
+                $sheetArray = $activeSheet->rangeToArray($sheetRange, null, false, false, false);
+
+                // 数据处理
+                if (!empty($sheetArray)) {
+
+                    $startIndex   = 1;
+                    $currentIndex = 1;
+
+                    if (1 == $currentRow) {
+                        $startIndex++;
+                        $header = $sheetArray[0];
+                    }
+
+                    foreach ($sheetArray as $row) {
+                        if ($currentIndex >= $startIndex) {
+                            foreach ($row as &$cell) {
+                                $cell = is_null($cell) ? '' : $cell;
+                            }
+                            $vals = [];
+                            $temp = array_combine($header, $row);
+                            foreach ($temp as $k => $v) {
+                                if (isset($fieldArr[$k]) && '' !== $k) {
+                                    $vals[$fieldArr[$k]] = $v;
+                                }
+                            }
+                            $insert[] = $vals;
+                        }
+                        $currentIndex++;
                     }
                 }
-                if ($row) {
-                    $insert[] = $row;
-                }
+
+                $chunkSheet->removeSheetByIndex(0);
+                $chunkSheet->createSheet(0);
+
             }
+
         } catch (Exception $exception) {
-            $this->error($exception->getMessage());
+            $msg = $exception->getMessage();
+            if (preg_match("/.*The filename (.+) is not recognised as an OLE file.*/is", $msg, $matches)) {
+                $msg = "读取文件失败, 建议重新将文件另存为csv、xls或xslx格式";
+            };
+            $this->error($msg);
         }
+
         if (!$insert) {
             $this->error(__('No rows were updated'));
         }
 
-        try {
-            //是否包含admin_id字段
-            $has_admin_id = false;
-            foreach ($fieldArr as $name => $key) {
-                if ($key == 'admin_id') {
-                    $has_admin_id = true;
-                    break;
-                }
+        //是否包含admin_id字段
+
+        $has_admin_id = false;
+        foreach ($fieldArr as $name => $key) {
+            if ('admin_id' == $key) {
+                $has_admin_id = true;
+                break;
             }
-            if ($has_admin_id) {
-                $auth = Auth::instance();
-                foreach ($insert as &$val) {
-                    if (!isset($val['admin_id']) || empty($val['admin_id'])) {
-                        $val['admin_id'] = $auth->isLogin() ? $auth->id : 0;
-                    }
-                }
-            }
-            $this->model->saveAll($insert);
-        } catch (PDOException $exception) {
-            $msg = $exception->getMessage();
-            if (preg_match("/.+Integrity constraint violation: 1062 Duplicate entry '(.+)' for key '(.+)'/is", $msg, $matches)) {
-                $msg = "导入失败，包含【{$matches[1]}】的记录已存在";
-            };
-            $this->error($msg);
-        } catch (Exception $e) {
-            $this->error($e->getMessage());
         }
 
-        $this->success();
+        if ($has_admin_id) {
+            $auth = Auth::instance();
+            foreach ($insert as &$val) {
+                if (!isset($val['admin_id']) || empty($val['admin_id'])) {
+                    $val['admin_id'] = $auth->isLogin() ? $auth->id : 0;
+                }
+            }
+        }
+
+        // 批量插入数据
+
+        $affectRows = 0;
+        $repeatRows = 0;
+
+        foreach ($insert as &$data) {
+
+            $errMsg = null;
+            $error  = null;
+            $event  = null;
+
+            $hook = [
+                'event' => 'before_import',
+                'class' => static::class,
+                'model' => $model,
+                'data'  => $data,
+                'error' => null,
+            ];
+
+            Hook::listen('table_import', $hook);
+
+            try {
+
+                $affectRows += (new $model($hook['data']))->save();
+                $event = 'after_import';
+
+            } catch (PDOException $e) {
+
+                $error = $e;
+                $event = 'pdo_exception';
+
+                if (preg_match("/.+Integrity constraint violation: 1062 Duplicate entry '(.+)' for key '(.+)'/is", $error, $matches)) {
+                    $errMsg = "导入失败，包含【{$matches[1]}】的记录已存在";
+                    $event  = 'pdo_duplicate_entry';
+                    $repeatRows++;
+                }
+
+            } catch (Exception $e) {
+                $error = $e;
+                $event = 'exception';
+            }
+
+            $hook['event'] = $event;
+            $hook['error'] = $error;
+
+            Hook::listen('table_import', $hook);
+            $error = $hook['error'];
+
+            if ($error) {
+                $errMsg = $errMsg ?: $error->getMessage();
+                $this->error($errMsg);
+            }
+
+        }
+
+        if (!$affectRows) {
+            $this->error(__('No rows were updated'));
+        }
+
+        $timer2 = microtime(true);
+        $usage2 = memory_get_usage();
+        $usage  = round(($usage2 - $usage1) / 1024 / 1024, 2);
+        $timer  = round($timer2 - $timer1, 3);
+
+        $success = vsprintf("导入成功!<br>%s<br>%s<br>%s<br>%s", [
+            sprintf("新增数据:  %d 条", $affectRows),
+            sprintf("重复数据:  %d 条", $repeatRows),
+            sprintf("内存占用:  %s MB", $usage),
+            sprintf("总共耗时:  %s 秒", $timer),
+        ]);
+
+        $this->success($success);
     }
 }