1. 修复批量查询导出bug

2. 修复配置文件生成路径bug

1. 增加导出文件可选类型 目前只支持xlsx和json
2. 增加中途退出保存功能，仅ctrl+c主动退出！
3. 增加实时文件写入功能，仅支持一个参数，支持投资查询

common/config.go

@@ -5,6 +5,7 @@ import (
 	"github.com/tidwall/gjson"
 	"github.com/wgpsec/ENScan/common/utils"
 	"go.mongodb.org/mongo-driver/bson/primitive"
+	"path/filepath"
 	"time"
 )
 
@@ -42,8 +43,10 @@ type ENOptions struct {
 	IsDebug        bool
 	IsJsonOutput   bool
 	Deep           int
-	IsMergeOut     bool //聚合
-	IsMerge        bool //聚合
+	UPOutFile      string
+	IsMergeOut     bool   //聚合
+	IsNoMerge      bool   //聚合
+	OutPutType     string // 导出文件类型
 	ENConfig       *ENConfig
 }
 
@@ -73,13 +76,13 @@ type ENsD struct {
 }
 
 func (h *ENOptions) GetDelayRTime() int64 {
+	if h.DelayTime == -1 {
+		return utils.RangeRand(1, 5)
+	}
 	if h.DelayTime != 0 {
 		h.DelayMaxTime = int64(h.DelayTime)
 	}
-	if h.DelayMaxTime == 0 {
-		return 0
-	}
-	return utils.RangeRand(1, h.DelayMaxTime)
+	return 0
 }
 
 func (h *ENOptions) GetENConfig() *ENConfig {
@@ -112,6 +115,7 @@ var DefaultAllInfos = []string{"icp", "weibo", "wechat", "app", "weibo", "job",
 var DefaultInfos = []string{"icp", "weibo", "wechat", "app", "wx_app"}
 var CanSearchAllInfos = []string{"enterprise_info", "icp", "weibo", "wechat", "app", "job", "wx_app", "copyright", "supplier", "invest", "branch", "holds", "partner"}
 var DeepSearch = []string{"invest", "branch", "holds", "supplier"}
+var ENSTypes = []string{"aqc", "tyc"}
 var ScanTypeKeys = map[string]string{
 	"aqc":     "爱企查",
 	"qcc":     "企查查",
@@ -141,9 +145,9 @@ type ENConfig struct {
 	}
 }
 
-var cfgYName = utils.GetConfigPath() + "/config.yaml"
-var cfgYV = 1.0
-var configYaml = `version: 1.0
+var cfgYName = filepath.Join(utils.GetConfigPath(), "config.yaml")
+var cfgYV = 0.4
+var configYaml = `version: 0.4
 cookies:
   aiqicha: ''           # 爱企查   Cookie
   tianyancha: ''        # 天眼查   Cookie
@@ -154,4 +158,5 @@ cookies:
   xlb: ''               # 小蓝本   Token
   qimai: ''             # 七麦数据  Cookie
   chinaz: ''			# 站长之家  Cookie
+  veryvp: '' 			# veryvp  Cookie
 `

common/flag.go

@@ -31,7 +31,6 @@ func Flag(Info *ENOptions) {
 	flag.StringVar(&Info.CompanyID, "i", "", "公司PID")
 	flag.StringVar(&Info.InputFile, "f", "", "批量查询，文本按行分隔")
 	flag.StringVar(&Info.ScanType, "type", "aqc", "查询渠道，可多选")
-	flag.StringVar(&Info.Output, "o", "", "结果输出的文件夹位置(默认为outs)")
 	//查询参数指定
 	flag.Float64Var(&Info.InvestNum, "invest", 0, "投资比例 eg 100")
 	flag.StringVar(&Info.GetFlags, "field", "", "获取字段信息 eg icp")
@@ -41,6 +40,9 @@ func Flag(Info *ENOptions) {
 	flag.BoolVar(&Info.IsGetBranch, "branch", false, "查询分支机构（分公司）信息")
 	flag.BoolVar(&Info.IsSearchBranch, "is-branch", false, "深度查询分支机构信息（数量巨大）")
 	flag.BoolVar(&Info.IsJsonOutput, "json", false, "json导出")
+	flag.StringVar(&Info.Output, "out-dir", "", "结果输出的文件夹位置(默认为outs)")
+	flag.StringVar(&Info.UPOutFile, "out-update", "", "导出指定范围文件，自更新")
+	flag.StringVar(&Info.OutPutType, "out-type", "xlsx", "导出的文件后缀 默认xlsx")
 	flag.BoolVar(&Info.IsDebug, "debug", false, "是否显示debug详细信息")
 	flag.BoolVar(&Info.IsShow, "is-show", true, "是否展示信息输出")
 	//其他设定
@@ -49,7 +51,7 @@ func Flag(Info *ENOptions) {
 	flag.IntVar(&Info.DelayTime, "delay", 0, "每个请求延迟（S）-1为随机延迟1-5S")
 	flag.StringVar(&Info.Proxy, "proxy", "", "设置代理")
 	flag.IntVar(&Info.TimeOut, "timeout", 1, "每个请求默认1（分钟）超时")
-	flag.BoolVar(&Info.IsMerge, "no-merge", false, "批量查询文件单独导出")
+	flag.BoolVar(&Info.IsNoMerge, "no-merge", false, "开启后查询文件将单独导出")
 	flag.BoolVar(&Info.Version, "v", false, "版本信息")
 	flag.Parse()
 }

common/output.go

@@ -10,13 +10,13 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"time"
 )
 
 // ENSMap 单独结构
 type ENSMap struct {
 	Name    string
-	Field   []string
 	JField  []string
 	KeyWord []string
 	Only    string
@@ -119,6 +119,26 @@ func InfoToMap(infos map[string][]gjson.Result, enMap map[string]*EnsGo, extraIn
 	return res
 }
 
+func OutStrByEnInfo(data map[string][]map[string]string, types string) (str string) {
+	var builder strings.Builder
+	s := data[types]
+	em := ENSMapLN[types].JField
+	for _, m := range s {
+		first := true
+		for _, key := range append(em, "from", "extra") {
+			if !first { // 如果不是第一个元素，则先写入逗号
+				builder.WriteString(",")
+			}
+			builder.WriteString(m[key])
+			first = false
+		}
+		builder.WriteString("\n")
+	}
+
+	str = builder.String()
+	return str
+}
+
 func OutFileByEnInfo(data map[string][]map[string]string, name string, types string, dir string) (err error) {
 	if dir == "!" {
 		gologger.Debug().Str("设定DIR", dir).Msgf("不导出文件")
@@ -158,8 +178,10 @@ func OutFileByEnInfo(data map[string][]map[string]string, name string, types str
 		for s, v := range data {
 			em := ENSMapLN[s]
 			exData := make([][]interface{}, len(v))
+			// 转换MAP格式为interface，进行excel写入
 			for i, m := range v {
 				if len(m) > 0 {
+					// 把信息全部提取出来，转为interface
 					for _, p := range append(em.JField, "from", "extra") {
 						exData[i] = append(exData[i], m[p])
 					}

common/parse.go

@@ -61,6 +61,11 @@ func Parse(options *ENOptions) {
 	if options.Output == "" {
 		options.Output = "outs"
 	}
+
+	if options.IsJsonOutput {
+		options.OutPutType = "json"
+	}
+
 	if options.Output == "!" {
 		gologger.Info().Msgf("当前模式不会导出文件信息！\n")
 	}
@@ -82,7 +87,7 @@ func Parse(options *ENOptions) {
 
 	//如果是指定全部数据
 	if options.ScanType == "all" {
-		options.GetType = []string{"aqc", "xlb", "qcc", "tyc", "qcc"}
+		options.GetType = ENSTypes
 		options.IsMergeOut = true
 	} else if options.ScanType != "" {
 		options.GetType = strings.Split(options.ScanType, ",")
@@ -111,6 +116,11 @@ func Parse(options *ENOptions) {
 			gologger.Fatal().Msgf("没有获取字段信息！\n" + options.GetFlags)
 		}
 	}
+
+	if options.UPOutFile != "" && len(options.GetField) > 1 {
+		gologger.Fatal().Msgf("自更新导出仅支持输出一个参数！⌈%s⌋", options.GetField)
+	}
+
 	// 是否深度获取分支机构
 	if options.IsSearchBranch {
 		options.IsGetBranch = true
@@ -135,10 +145,10 @@ func Parse(options *ENOptions) {
 	}
 	options.GetField = utils.SetStr(options.GetField)
 
-	if options.IsMerge == true {
+	if options.IsNoMerge {
 		gologger.Info().Msgf("批量查询文件将单独导出！\n")
 	}
-	options.IsMergeOut = !options.IsMerge
+	options.IsMergeOut = !options.IsNoMerge
 	options.GetField = utils.SetStr(options.GetField)
 	options.ENConfig = conf
 }

common/utils/excel.go

@@ -8,6 +8,7 @@ package utils
  **/
 
 import (
+	"fmt"
 	"github.com/xuri/excelize/v2"
 	"strconv"
 )
@@ -21,7 +22,13 @@ const maxCharCount = 26
 // rows 数据切片，是一个二维数组
 // f 为了创建多个单元格，得把标签往外放
 func ExportExcel(sheetName string, headers []string, rows [][]interface{}, f *excelize.File) (*excelize.File, error) {
-	sheetIndex := f.NewSheet(sheetName)
+	sheetIndex, err := f.GetSheetIndex(sheetName)
+	if sheetIndex == -1 {
+		sheetIndex, err = f.NewSheet(sheetName)
+	}
+	if err != nil {
+		return nil, err
+	}
 	maxColumnRowNameLen := 1 + len(strconv.Itoa(len(rows)))
 	columnCount := len(headers)
 	if columnCount > maxCharCount {
@@ -81,3 +88,60 @@ func getColumnRowName(columnName []byte, rowIndex int) (columnRowName string) {
 	columnName = columnName[:l]
 	return
 }
+
+func StreamWriterFunc(contents [][]string) {
+	//打开工作簿
+	file, err := excelize.OpenFile("Book1.xlsx")
+	if err != nil {
+		return
+	}
+	sheet_name := "Sheet1"
+	//获取流式写入器
+	streamWriter, _ := file.NewStreamWriter(sheet_name)
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	rows, _ := file.GetRows(sheet_name) //获取行内容
+	cols, _ := file.GetCols(sheet_name) //获取列内容
+	fmt.Println("行数rows:  ", len(rows), "列数cols:  ", len(cols))
+
+	//将源文件内容先写入excel
+	for rowid, row_pre := range rows {
+		row_p := make([]interface{}, len(cols))
+		for colID_p := 0; colID_p < len(cols); colID_p++ {
+			//fmt.Println(row_pre)
+			//fmt.Println(colID_p)
+			if row_pre == nil {
+				row_p[colID_p] = nil
+			} else {
+				row_p[colID_p] = row_pre[colID_p]
+			}
+		}
+		cell_pre, _ := excelize.CoordinatesToCellName(1, rowid+1)
+		if err := streamWriter.SetRow(cell_pre, row_p); err != nil {
+			fmt.Println(err)
+		}
+	}
+
+	//将新加contents写进流式写入器
+	for rowID := 0; rowID < len(contents); rowID++ {
+		row := make([]interface{}, len(contents[0]))
+		for colID := 0; colID < len(contents[0]); colID++ {
+			row[colID] = contents[rowID][colID]
+		}
+		cell, _ := excelize.CoordinatesToCellName(1, rowID+len(rows)+1) //决定写入的位置
+		if err := streamWriter.SetRow(cell, row); err != nil {
+			fmt.Println(err)
+		}
+	}
+
+	//结束流式写入过程
+	if err := streamWriter.Flush(); err != nil {
+		fmt.Println(err)
+	}
+	//保存工作簿
+	if err := file.SaveAs("Book1.xlsx"); err != nil {
+		fmt.Println(err)
+	}
+}

common/utils/utils.go

@@ -96,7 +96,7 @@ func DelInList(target string, list []string) []string {
 	return result
 }
 
-func ReadFile(filename string) []string {
+func ReadFileOutLine(filename string) []string {
 	var result []string
 	if FileExists(filename) {
 		f, err := os.Open(filename)
@@ -115,7 +115,6 @@ func ReadFile(filename string) []string {
 		_ = f.Close()
 	}
 	result = SetStr(result)
-	gologger.Info().Msgf("读取到 %d 条信息（已去重）\n", len(result))
 	return result
 }
 

enscan.go

@@ -3,6 +3,7 @@ package main
 import (
 	"github.com/wgpsec/ENScan/common"
 	"github.com/wgpsec/ENScan/common/gologger"
+	"github.com/wgpsec/ENScan/common/utils"
 	"github.com/wgpsec/ENScan/runner"
 	"log"
 	"os"
@@ -16,13 +17,15 @@ func main() {
 		for {
 			select {
 			case <-quitSig:
-				if !runner.CurDone {
-					gologger.Error().Msgf("任务未完成退出，自动保存文件！数据长度 %d", len(runner.TmpData))
-					rdata := common.InfoToMap(runner.TmpData, runner.CurrJob.GetENMap(), "")
-					err := common.OutFileByEnInfo(rdata, "意外退出保存文件", "xlsx", "outs")
-					if err != nil {
-						gologger.Error().Msgf(err.Error())
-					}
+				gologger.Error().Msgf("任务未完成退出，自动保存过程文件！")
+				enDataList := make(map[string][]map[string]string)
+				close(runner.EnCh)
+				for ch := range runner.EnCh {
+					utils.MergeMap(ch, enDataList)
+				}
+				err := common.OutFileByEnInfo(enDataList, "意外退出保存文件", "xlsx", "outs")
+				if err != nil {
+					gologger.Error().Msgf(err.Error())
 				}
 				log.Fatal("exit.by.signal")
 			}

go.mod

@@ -3,24 +3,24 @@ module github.com/wgpsec/ENScan
 go 1.21
 
 require (
-	github.com/antchfx/htmlquery v1.2.4
+	github.com/antchfx/htmlquery v1.3.1
 	github.com/go-resty/resty/v2 v2.7.0
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/projectdiscovery/gologger v1.1.12
 	github.com/robertkrimen/otto v0.0.0-20211024170158-b87d35c0b86f
 	github.com/tidwall/gjson v1.11.0
 	github.com/tidwall/sjson v1.2.3
-	github.com/xuri/excelize/v2 v2.4.1
+	github.com/xuri/excelize/v2 v2.8.1
 	go.mongodb.org/mongo-driver v1.9.0
-	golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b
+	golang.org/x/net v0.21.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
 require (
 	github.com/andybalholm/brotli v1.0.6 // indirect
-	github.com/antchfx/xpath v1.2.0 // indirect
+	github.com/antchfx/xpath v1.3.0 // indirect
 	github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
-	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.13.6 // indirect
@@ -33,15 +33,16 @@ require (
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/nwaples/rardecode v1.1.3 // indirect
 	github.com/pierrec/lz4/v4 v4.1.2 // indirect
-	github.com/richardlehane/mscfb v1.0.3 // indirect
-	github.com/richardlehane/msoleps v1.0.1 // indirect
+	github.com/richardlehane/mscfb v1.0.4 // indirect
+	github.com/richardlehane/msoleps v1.0.3 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/ulikunitz/xz v0.5.11 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
-	github.com/xuri/efp v0.0.0-20210322160811-ab561f5b45e3 // indirect
-	golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect
-	golang.org/x/text v0.3.7 // indirect
+	github.com/xuri/efp v0.0.0-20231025114914-d1ff6096ae53 // indirect
+	github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05 // indirect
+	golang.org/x/crypto v0.19.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	gopkg.in/djherbis/times.v1 v1.3.0 // indirect
 	gopkg.in/sourcemap.v1 v1.0.5 // indirect
 )