diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..38ac6d5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,152 @@ +### macOS template +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### Windows template +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +### Go template +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +### Example user template template +### Example user template + +# IntelliJ project files +.idea +*.iml +out +gen +### JetBrains template +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + diff --git a/bin/koala b/bin/koala old mode 100644 new mode 100755 index 7dd92e2..8ce060e Binary files a/bin/koala and b/bin/koala differ diff --git a/cmd/main.go b/cmd/main.go new file mode 100644 index 0000000..8160a46 --- /dev/null +++ b/cmd/main.go @@ -0,0 +1,41 @@ +/** + * Koala Rule Engine Core + * + * @package: main + * @desc: koala engine - Main router + * + * koala频率控制服务 (规则引擎) + * 用途：是为了解决用户提交等相关频率控制的一个通用服务，主要是为了替换传统写死代码的频率控制模块以达到 高性能、灵活配置的要求。 + * 方案：支持高度灵活的规则配置；并实现了规则配置的动态加载； 后端cache采用带连接池的redis。 + * + * @author: heiyeluren + * @github: https://github.com/heiyeluren + * @blog: https://blog.csdn.net/heiyeshuwu + * + */ + +package main + +import ( + "github.com/heiyeluren/koala/koala" +) + +/** + * koala服务进程的main函数 + */ +func main() { + // Start koala + koala.Run() + + // 启动 rule统计协程 + go koala.CounterAgent() + + // 启动 规则更新协程，定期检查 policy 更新 + go koala.PolicyLoader() + + // 启动 http监听协程 + go koala.FrontListen() + + // hold 住 main协程 + select {} +} diff --git a/conf/koala.conf b/conf/koala.conf index 5c3c04f..9972ca5 100644 --- a/conf/koala.conf +++ b/conf/koala.conf @@ -26,10 +26,10 @@ externalWriteTimeout = 500 #-------------- #redis服务地址 -redis_server = 127.0.0.1:3721 +redis_server = 127.0.0.1:6379 #redis服务password -redis_auth = Yhs8jCfcys +redis_auth = #redis连接池，最大空闲连接数 redis_pool_maxIdle = 5 @@ -52,10 +52,10 @@ log_warning_file_path = log/koala.log.wf log_cron_time = day #日志文件生存周期, 单位:天 -log_file_ilfe_time = 7 +log_file_life_time = 7 -#日志chan队列的buffer长度，建议不要少于10240，不建议多于1024000，最长：67021478（超过这个值会无法启动） -log_chan_buff_size = 10240 +#日志channel队列的buffer长度，建议不要少于10240，不建议多于1024000，最长：67021478（超过这个值会无法启动） +log_channel_buff_size = 10240 #日志刷盘的时间间隔，单位：毫秒，建议500~5000毫秒，建议不超过30秒 log_flush_timer = 1000 diff --git "a/doc/Koala\351\205\215\347\275\256\346\226\207\344\273\266\346\240\267\345\274\217.txt" "b/doc/Koala\351\205\215\347\275\256\346\226\207\344\273\266\346\240\267\345\274\217.txt" index 614e4c4..54066c0 100644 --- "a/doc/Koala\351\205\215\347\275\256\346\226\207\344\273\266\346\240\267\345\274\217.txt" +++ "b/doc/Koala\351\205\215\347\275\256\346\226\207\344\273\266\346\240\267\345\274\217.txt" @@ -1,243 +1,241 @@ - -############################################ -# -# Koala �����ļ� -# -############################################ - - - -###################### -# �ʱ��ļ� -###################### -[dicts] -global_qid_whitelist : /home/q/koala/data/global_qid_whitelist.dat -global_qid_blacklist : /home/q/koala/data/global_qid_blacklist.dat - -global_ip_whitelist : /home/q/koala/data/global_ip_whitelist.dat -global_ip_blacklist : /home/q/koala/data/global_ip_blacklist.dat - - -###################### -# �����б� -###################### -[rules] - -#--------- -# ���� -#--------- -#ȫ�ְ����� -rule : [count] [qid-IN-global_qid_whitelist] [time=1; count=0;] [result=1; return=101] -rule : [count] [ip-IN-global_ip_whitelist] [time=1; count=0;] [result=1; return=102] - -#ȫ�ֺ����� -rule : [count] [qid-IN-global_qid_blacklist] [time=1; count=0;] [result=2; return=103] -rule : [count] [ip-IN-global_ip_blacklist] [time=1; count=0;] [result=2; return=104] - -#��ע���û��ύǿ�Ƴ�4λ��֤�� -rule : [count] [act=add_ask,add_answer;is_new=1;qid=+ ] [time=1; count=0;] [result=3; return=110] - -#ÿ�죬����qid�û������ʺͻش�����������ܳ���1000�� -rule : [count] [act=add_ask,add_answer{*};qid=+] [time=86400; count=1000;] [result=2; return=111] - -#ÿ�죬qid���ִ���200000000���ˣ�ÿ�����ʻش�������ֹ����10(���Խ��ij���Ժ�ע���û����������) -rule : [count] [act=add_ask,add_answer{*};qid>200000000] [time=86400; count=10;] [result=2; return=112] - -#���ʣ�ͬip�û����������ʳ���30���Ժ�ÿ�����ʼ��2�� -rule : [count] [act=add_ask,add_answer{*};qid>200000000] [time=86400; count=10;] [result=2; return=112] - - -#--------- -# ���� -#--------- -#���ʣ�ͬqid��2�������ʲ��ܳ���1�Σ�ÿ�����ʼ��2�룩 -rule : [count] [act=add_ask;qid=1-2147483647;] [time=2; count=1;] [result=2; return=201] - -#���ʣ�ͬqid��ÿ�����ʲ�����500�� -rule : [count] [act=add_ask;qid=1-2147483647;] [time=86400; count=500;] [result=2; return=202] - -#�������ʣ�ͬqidһ�����20�� -rule : [count] [act=add_ask;qid=+;is_at=1;] [time=86400; count=20;] [result=2; return=203] - -#�������ʣ���ij�˶������ʣ�ͬqidһ�����5�� -rule : [count] [act=add_ask;qid=+;is_at=1;to_qid=+;] [time=86400; count=5;] [result=2; return=204] - -#���ʣ�ͬqidһ�쳬��10�κ����֤�� -rule : [count] [act=add_ask;qid=+;] [time=86400; count=10;] [result=3; return=205] - -#���ʣ�ͬIP��ÿ�����ʲ�����1000�� -rule : [count] [act=add_ask;ip=+;] [time=86400; count=1000;] [result=2; return=220] - -#���ʣ�ͬIP��һ�쳬��30�γ���֤�� -rule : [count] [act=add_answer;ip=+;] [time=86400; count=30;] [result=3; return=221] - -#���ʣ�ͬIP��2�������ʲ�����30�� -rule : [count] [act=add_ask ip=+;] [time=2; count=30;] [result=2; return=222] - -#���ʣ�ͬIP��10�������ʲ�����50�� -rule : [count] [act=add_ask;ip=+;] [time=10; count=50;] [result=2; return=223] - -#���ʣ�ͬIP������30�κ�ÿ�����ʼ��2�� -rule : [base] [act=add_ask;ip=+;] [base=30; time=2; count=1;] [result=2; return=224] - -#���ʣ�ͬIP������50�κ�ÿ�����ʼ��10�� -rule : [base] [act=add_ask;ip=+;] [base=50; time=10; count=1;] [result=2; return=225] - - -#--------- -# �ش� -#--------- - -#�ش�ͬqidÿ��2000 -rule : [count] [act=add_answer;qid=+;] [time=86400; count=2000;] [result=2; return=301] - -#�ش�ͬһqidһ�쳬��200�γ���֤�� -rule : [count] [act=add_answer;qid=+;] [time=86400; count=200;] [result=3; return=302] - -#�ش�ͬһipһ�쳬��1000�γ���֤�� -rule : [count] [act=add_answer;ip=+;] [time=86400; count=1000;] [result=3; return=320] - -#ר�һش�ͬqidÿ��200000 -rule : [count] [act=add_answer;qid=+;is_at=1] [time=86400; count=200000;] [result=2; return=321] - -#�ش�ͬipÿ��5000 -rule : [count] [act=add_answer;ip=+;] [time=86400; count=5000;] [result=2; return=322] - -#------------ -# �а��� -#------------ -#�а�����ͬipһ��50�� -rule : [count] [act=add_help;ip=+;] [time=86400; count=50;] [result=2; return=420] - -#�а�����ͬip��ͬ1����һ��1�� -rule : [count] [act=add_help;ip=+;ask_id=+] [time=86400; count=1;] [result=2; return=421] - - -####################### -# ���ؽ������ -####################### -[result] - -#Ĭ�Ϲ��� (Ĭ����ͨ��״̬) -0 : { "ret_type":0, "ret_code" : 0, "err_no":0, "err_msg":"", "str_reason":"Allow", "need_vcode":0, "vcode_len":4, "vcode_type":0, "other":"", "version":0 } - -#ͨ�� (û�������κο��ƣ�ͨ�ù���) -1 : { "ret_type":1, "ret_code" : 0, "err_no":0, "err_msg":"", "str_reason":"Allow", "need_vcode":0, "vcode_len":4, "vcode_type":0, "other":"", "version":0 } - -#��ͨ�� (������Ƶ�ʿ��ƣ�������֤�룬ֱ�ӽ�ֹ����) -2 : { "ret_type":2, "ret_code" : 0, "err_no":10, "err_msg":"", "str_reason":"Deny", "need_vcode":0, "vcode_len":4, "vcode_type":0, "other":"", "version":0 } - -#��4λ��֤�� -3 : { "ret_type":3, "ret_code" : 0, "err_no":20, "err_msg":"", "str_reason":"Vcode", "need_vcode":1, "vcode_len":4, "vcode_type":0, "other":"", "version":0 } - -#��6λ��֤�� -4 : { "ret_type":4, "ret_code" : 0, "err_no":21, "err_msg":"", "str_reason":"Vcode", "need_vcode":1, "vcode_len":6, "vcode_type":0, "other":"", "version":0 } - - - - - - - -###################################### ����˵���ָ��� ######################################## -# -# ��Koala �����ļ�ʹ��˵���� -# -# ������Ҫ��Ϊ�������֣� -# [dicts] : �ʱ����� -# [rules] : �������� -# [result]: ���ؽ������ -# -# [dicts]�ʱ����ã� -# ��Ҫ����ͨ��ð�� : ���ֵ��������࣬�����������ļ��ĵ����֣���Ҫ���ṩ��rule����ʹ�õģ��ļ����ݱ�����һ��һ�������ݣ���Ҫ�Ǹ�ij���������бȶԣ��Ҳ���ֵ����ļ�·������Ҫ������ʱ��ļ�ʵ���ڵ�ǰ��������·���� -# ʾ���� -# global_qid_whitelist : /home/q/koala/data/global_qid_whitelist.dat -# ��˵һ������ global_qid_whitelist �Ĵʱ��ļ���·���� /home/q/koala/data/global_qid_whitelist.dat -# -# -# [rules]�������ã� -# !!ע�⣺�������������Ǵ��ϵ��£�����ǰ��Ĺ������ȼ����ں���Ĺ������ǰ��������о�ֱ�ӷ��أ����˼�����Ĺ���!! -# -# rule���ñȽϸ��ӣ���ҪҲ�Ƿ�Ϊ���顣 -# ð������ rule �Ǵ�������һ���������ã�ð���Ҳ���4������ [] �������ֵĵ����ݼ��� -# ��һ��[]�� ��Ҫ�DZ�ʾ����������ʲô���͵ģ�Ŀǰ֧�� [count] �� [base] �������� -# [count] ���;���һ���׼�ļ������� -# [base] �������������ͣ�һ���������ơ����ʲ�������30���Ժ��û�ÿ��2������ύ�����ֿ��ƣ������ڵ������ֶ��л���Ҫ����base�ֶ� -# �ڶ���[]���м���ǿͻ��˴��ݲ������ϣ������ṹ�ǣ�key=value; ����ʽ��ÿ�������м�ʹ�÷ֺ����֡����value��ȡֵ�������¼��֣� -# ����ȡֵ�� -# ���� act=add_answer������˵���key��act�ģ�ֵ�ǹ̶���add_answer�������б�key���������� -# ���� act!=add_answer, ����˵ֻҪ�������������act�ֶΣ����Ҳ����� add_answer �������� -# -# ��Χֵ����Χֵ���� ���䡢�����и�ĵ����������ݣ� -# qid=10,23,45 : ����˵���qid������3�����ֲ������У���֧���ַ������ͣ��ַ���������Ҫ���ڴʱ��ļ��� -# qid=1-2147483647 : ����˵qid��ȡֵ��Χ�� 1 ~ 2147483647 ���ж��� -# qid>10000 : ��˵qid����10000�����з�Χ���� -# ip=192.168.0.* : ��˵ǰ�����ε�ַһ�¾������У���ͬ�ڣ�192.168.0.1-192.168.0.256��Ҳ����ʹ�� 192.168.*.* ���ָ���Χ�� -# ip=192.168.0.1/24 : ����ʹ��ip��ַ���εķ�ʽҲ��֧�ֵģ�����������ʽ�� -# ip=192.168.0.1-192.168.0.256 : ����һ��IP��ַ�ķ�Χֵ���������ip��ַ��Χ��ip��ַ��Χ��������*��ͨ�� -# qid-IN-global_qid_whitelist : ���������˵�������õ��ļ��б���ֻҪ��ǰqid���������У��������У���ͬ�� qid=10,23,45 �������ã�ֻ�ǿ������Ӻܶ� -# ip=192.168.0.1-192.168.0.256,10.0.24.*,202.106.34.21/24 ��˵�����η�Χ�ڵ�ip�����б����� -# -# -# ���������ͣ� -# ���� # : �������������DZ�ע�͵ģ�һ����������� -# ������ [] : ��Ҫ����Ϊij��β����ı�ʾ����[rule]���̶ֹ���group�����Ų�ͬ -# �ֺ� ; : ������������ֱ�ӽ����� -# �Ⱥ� = : ����key��value�ṹ�ĵ��ڸ�ֵ�IJ��� -# ������ != : �������������value�����������key���Ǹ��Ƚϲ��� -# ���� , : �����ж����ֵ���߷�Χֵ����Ҫʹ�ö���һһ��������һ�����򡱲��� -# �Ӻ� + : �����������ֶΰ���������ֵ��ʲô����Ҫ������һ��Ҫ������ֵ -# �Ǻ� * : ��Ҫ����IP��ַ����������ͨ����IJ��� -# ð�� : : ��Ҫ���ڸ�ֵ����ֶ����ֵ���; -# ���� {*} : ��һ���������������Ҫ������value���棬��� act=add_answer,add_ask{*} ����һ���������ǰ������������ϲ���һ�����������ǽ���ͳһ�������������� act=add_answer,add_ask �����Ƿֳ����������ģ�ʹ��{*} ���ܹ��ϲ���һ�������ˣ����ijЩ������Ҫ�ļ������� -# ���� - : Ŀǰ��Ҫ�����ڷ�Χ��������� 1-100����˵1��100����������У������� qid-IN-xxx_list Ҳ��˵qid��xxx_list��ʱ������� -# ���� > : ��Χ���У�һ���������ַ�Χ���棬���� qid>100000����˵qid��Χ����100000������ -# С�� < : ��Χ���У����ƴ��ڣ�������С�ڵIJ��������� qid<10000����˵qidС��10000�Ķ����� -# ���� IN : ��Ҫ�ǻ���һ���ʱ�������û���������ʱ���˵������������ʱ����������С�������ƣ�qid-IN-xxx_list -# ������ NOTIN : Ҳ����һ���ʱ������������˵����û�û����������ʱ���˵��ͨ���������������ƣ�qid-NOTIN-xxx_list -# -# -# (ע�⣬ֻ�����κ�IP��ַ���Ͳ�֧�ַ�Χ���ַ�����������һ���ʱ��ļ�����) -# -# ������[]�� �����Ǵ������Dz������ж���������ʽ�ǹ̶��ġ� -# -# ���� [time=86400; count=50;] [result=2; return=401] -# base : һ��������[base]���͵�rule����Ҫ������������һ����������������˵������base��Ȼ��Ż�ȥ���к������Щ�����жϡ�Ӧ�ó������ƣ����ʲ�������30���Ժ��û�ÿ��2������ύ�������ѡ��ֵ��0����û�����ã�������ͨ�������С� -# time : ��ʱ�䣬��λ���� -# count : �DZ��������޵IJ���������0�Ǵ������ޣ��������Dz��������� -# -# ������[]�����õ������������ж������ķ��ؽ������ʽ�ǹ̶��� -# result : ��˵����������[result]�����õ��ĸ���� -# return : ��˵��ǰ������򷵻ظ����÷���һ�����������������������ݻ�ѹ������ result �ﷵ�ظ����÷� -# -# #ʾ�������ʣ�ͬһqidһ�쳬��10�κ����֤�� -# rule : [count] [act=add_ask;qid=+;] [time=86400; count=10;] [result=3; return=208] -# -# #ʾ�������ʣ�ͬIP������30�κ�ÿ�����ʼ��2�� -# rule : [base] [act=add_ask;ip=+;] [base=30; time=2; count=1;] [result=2; return=204] -# -# -# -# [result]���ؽ�����ã� -# ���ؽ�����ñȽϼ򵥣���ҪҲ�Ƿ�Ϊ2�顣 -# ð������ ���� �Ǵ�������һ�����ؽ�����ã�ð���Ҳ������Ƿ��ظ�ǰ�˵�json��ʽ���ݡ� -# �Ҳ�json��ʽ�����ǹ̶��ģ����Ƹ�ʽ�� -# 0 : { "ret_type":0, "ret_code" : 0, "err_no":0, "err_msg":"", "str_reason":"Allow", "need_vcode":0, "vcode_len":4, "vcode_type":0, "other":"", "version":0 } -# -# �ֶ������� -# ret_type : ��result ������ -# ret_code : �ǹ���������� return ��������ǰ�˿����ڻ�ȡ����һЩ��ʾ���������߼�¼���� -# err_no : �����(һ���д�������) -# err_msg : ������Ϣ(һ�������������Ϣ) -# str_reason : ���������Ϣ(�ο�ʹ��) -# need_vcode : �Ƿ����֤�룬0������1�� -# vcode_len : ��֤�볤�ȣ�һ����4λ(Ԥ���ֶΣ���Ҫ����֤���Ƿ�֧��) -# vode_type : ��֤�����ͣ���������ͨ��֤�롢������֤�롢������֤�롢�Ź�����֤��� -# other : Ԥ���ֶ� -# version : ��ǰ������Ϣ�İ汾��Ԥ���ֶ� -# -# -# -# -###################################### ����˵�������� ######################################## - +############################################ +# +# Koala 配置文件 +# +############################################ + + + +###################### +# 词表文件 +###################### +[dicts] +global_qid_whitelist : /home/q/koala/data/global_qid_whitelist.dat +global_qid_blacklist : /home/q/koala/data/global_qid_blacklist.dat + +global_ip_whitelist : /home/q/koala/data/global_ip_whitelist.dat +global_ip_blacklist : /home/q/koala/data/global_ip_blacklist.dat + + +###################### +# 规则列表 +###################### +[rules] + +#--------- +# 基础 +#--------- +#全局白名单 +rule : [count] [qid-IN-global_qid_whitelist] [time=1; count=0;] [result=1; return=101] +rule : [count] [ip-IN-global_ip_whitelist] [time=1; count=0;] [result=1; return=102] + +#全局黑名单 +rule : [count] [qid-IN-global_qid_blacklist] [time=1; count=0;] [result=2; return=103] +rule : [count] [ip-IN-global_ip_blacklist] [time=1; count=0;] [result=2; return=104] + +#新注册用户提交强制出4位验证码 +rule : [count] [act=add_ask,add_answer;is_new=1;qid=+ ] [time=1; count=0;] [result=3; return=110] + +#每天，单个qid用户，提问和回答操作总数不能超过1000次 +rule : [count] [act=add_ask,add_answer{*};qid=+] [time=86400; count=1000;] [result=2; return=111] + +#每天，qid数字大于200000000的人，每天提问回答数量禁止超过10(可以解决某天以后注册用户的特殊操作) +rule : [count] [act=add_ask,add_answer{*};qid>200000000] [time=86400; count=10;] [result=2; return=112] + +#提问，同ip用户在连续提问超过30次以后，每次提问间隔2秒 +rule : [count] [act=add_ask,add_answer{*};qid>200000000] [time=86400; count=10;] [result=2; return=112] + + +#--------- +# 提问 +#--------- +#提问，同qid，2秒内提问不能超过1次（每次提问间隔2秒） +rule : [count] [act=add_ask;qid=1-2147483647;] [time=2; count=1;] [result=2; return=201] + +#提问，同qid，每天提问不超过500次 +rule : [count] [act=add_ask;qid=1-2147483647;] [time=86400; count=500;] [result=2; return=202] + +#定向提问，同qid一天最多20次 +rule : [count] [act=add_ask;qid=+;is_at=1;] [time=86400; count=20;] [result=2; return=203] + +#定向提问，向某人定向提问，同qid一天最多5次 +rule : [count] [act=add_ask;qid=+;is_at=1;to_qid=+;] [time=86400; count=5;] [result=2; return=204] + +#提问，同qid一天超过10次后出验证码 +rule : [count] [act=add_ask;qid=+;] [time=86400; count=10;] [result=3; return=205] + +#提问，同IP，每天提问不超过1000次 +rule : [count] [act=add_ask;ip=+;] [time=86400; count=1000;] [result=2; return=220] + +#提问，同IP，一天超过30次出验证码 +rule : [count] [act=add_answer;ip=+;] [time=86400; count=30;] [result=3; return=221] + +#提问，同IP，2秒内提问不超过30次 +rule : [count] [act=add_ask ip=+;] [time=2; count=30;] [result=2; return=222] + +#提问，同IP，10秒内提问不超过50次 +rule : [count] [act=add_ask;ip=+;] [time=10; count=50;] [result=2; return=223] + +#提问，同IP，超过30次后，每次提问间隔2秒 +rule : [base] [act=add_ask;ip=+;] [base=30; time=2; count=1;] [result=2; return=224] + +#提问，同IP，超过50次后，每次提问间隔10秒 +rule : [base] [act=add_ask;ip=+;] [base=50; time=10; count=1;] [result=2; return=225] + + +#--------- +# 回答 +#--------- + +#回答，同qid每天2000 +rule : [count] [act=add_answer;qid=+;] [time=86400; count=2000;] [result=2; return=301] + +#回答，同一qid一天超过200次出验证码 +rule : [count] [act=add_answer;qid=+;] [time=86400; count=200;] [result=3; return=302] + +#回答，同一ip一天超过1000次出验证码 +rule : [count] [act=add_answer;ip=+;] [time=86400; count=1000;] [result=3; return=320] + +#专家回答，同qid每天200000 +rule : [count] [act=add_answer;qid=+;is_at=1] [time=86400; count=200000;] [result=2; return=321] + +#回答，同ip每天5000 +rule : [count] [act=add_answer;ip=+;] [time=86400; count=5000;] [result=2; return=322] + +#------------ +# 有帮助 +#------------ +#有帮助，同ip一天50次 +rule : [count] [act=add_help;ip=+;] [time=86400; count=50;] [result=2; return=420] + +#有帮助，同ip对同1问题一天1次 +rule : [count] [act=add_help;ip=+;ask_id=+] [time=86400; count=1;] [result=2; return=421] + + +####################### +# 返回结果配置 +####################### +[result] + +#默认规则 (默认是通过状态) +0 : { "ret_type":0, "ret_code" : 0, "err_no":0, "err_msg":"", "str_reason":"Allow", "need_vcode":0, "vcode_len":4, "vcode_type":0, "other":"", "version":0 } + +#通过 (没有命中任何控制，通用规则) +1 : { "ret_type":1, "ret_code" : 0, "err_no":0, "err_msg":"", "str_reason":"Allow", "need_vcode":0, "vcode_len":4, "vcode_type":0, "other":"", "version":0 } + +#不通过 (命中了频率控制，不出验证码，直接禁止操作) +2 : { "ret_type":2, "ret_code" : 0, "err_no":10, "err_msg":"", "str_reason":"Deny", "need_vcode":0, "vcode_len":4, "vcode_type":0, "other":"", "version":0 } + +#出4位验证码 +3 : { "ret_type":3, "ret_code" : 0, "err_no":20, "err_msg":"", "str_reason":"Vcode", "need_vcode":1, "vcode_len":4, "vcode_type":0, "other":"", "version":0 } + +#出6位验证码 +4 : { "ret_type":4, "ret_code" : 0, "err_no":21, "err_msg":"", "str_reason":"Vcode", "need_vcode":1, "vcode_len":6, "vcode_type":0, "other":"", "version":0 } + + + + + + + +###################################### 配置说明分割线 ######################################## +# +# 【Koala 配置文件使用说明】 +# +# 配置主要分为三个部分： +# [dicts] : 词表配置 +# [rules] : 规则配置 +# [result]: 返回结果配置 +# +# [dicts]词表配置： +# 主要就是通过冒号 : 区分的左右两侧，左侧的是数据文件的的名字，主要是提供给rule规则使用的，文件内容必须是一行一条的内容，主要是跟某个参数进行比对，右侧出现的是文件路径，主要是这个词表文件实际在当前服务器的路径。 +# 示例： +# global_qid_whitelist : /home/q/koala/data/global_qid_whitelist.dat +# 是说一个叫做 global_qid_whitelist 的词表文件，路径是 /home/q/koala/data/global_qid_whitelist.dat +# +# +# [rules]规则配置： +# !!注意：整个规则运行是从上到下，排在前面的规则优先级高于后面的规则，如果前面规则命中就直接返回，不顾及后面的规则。!! +# +# rule配置比较复杂，主要也是分为三块。 +# 冒号左侧的 rule 是代表这是一条规则配置，冒号右侧是4组依赖 [] 进行区分的的数据集。 +# 第一组[]： 主要是表示这条规则是什么类型的，目前支持 [count] 和 [base] 两种类型 +# [count] 类型就是一般标准的计数类型 +# [base] 这种是特殊类型，一般满足类似“提问操作超过30次以后，用户每隔2秒才能提交”这种控制，另外在第三组字段中还需要增加base字段 +# 第二组[]：中间的是客户端传递参数集合，基本结构是：key=value; 的形式，每个参数中间使用分号区分。这个value的取值包括如下几种： +# 绝对取值： +# 比如 act=add_answer，就是说这个key是act的，值是固定是add_answer才算命中本key，否则不命中 +# 比如 act!=add_answer, 就是说只要这个操作包含了act字段，并且不等于 add_answer 都算命中 +# +# 范围值：范围值包括 区间、逗号切割的单独多条数据， +# qid=10,23,45 : 就是说这个qid包含这3个数字才算命中，不支持字符串类型，字符串类型需要放在词表文件中 +# qid=1-2147483647 : 就是说qid的取值范围从 1 ~ 2147483647 命中都算 +# qid>10000 : 是说qid大于10000的所有范围命中 +# ip=192.168.0.* : 是说前面三段地址一致就算命中，等同于：192.168.0.1-192.168.0.256，也可以使用 192.168.*.* 这种更大范围的 +# ip=192.168.0.1/24 : 这种使用ip地址区段的方式也是支持的（子网掩码形式） +# ip=192.168.0.1-192.168.0.256 : 这是一个IP地址的范围值，代表这个ip地址范围，ip地址范围还可以用*来通配 +# qid-IN-global_qid_whitelist : 这个配置是说上面配置的文件列表，只要当前qid包含在其中，就算命中，等同于 qid=10,23,45 这种配置，只是可以增加很多 +# ip=192.168.0.1-192.168.0.256,10.0.24.*,202.106.34.21/24 是说这三段范围内的ip都命中本操作 +# +# +# 操作符类型： +# 井号 # : 代表本行内容是被注释的，一般出现在行首 +# 中括号 [] : 主要是作为某大段操作的表示，跟[rule]这种固定的group中括号不同 +# 分号 ; : 代表两个参数直接结束了 +# 等号 = : 代表key、value结构的等于赋值的操作 +# 不等于 != : 代表不满足这个value就是命中这个key，是个比较操作 +# 逗号 , : 代表有多个数值或者范围值，需要使用逗号一一描述，是一个“或”操作 +# 加号 + : 代表本参数字段包含，具体值是什么不重要，就是一定要包含本值 +# 星号 * : 主要是在IP地址等数据里面通配符的操作 +# 冒号 : : 主要是在赋值大块字段区分的用途 +# 特殊 {*} : 这一个特殊操作符，主要是用在value后面，针对 act=add_answer,add_ask{*} 这样一个操作，是把这两个操作合并成一个计数，我们叫做统一计数操作，对于 act=add_answer,add_ask 我们是分成两个计数的，使用{*} 就能够合并成一个计数了，解决某些特殊需要的计数场合 +# 减号 - : 目前主要是用在范围操作里，比如 1-100，是说1到100这个区间命中，或者是 qid-IN-xxx_list 也是说qid在xxx_list这词表中命中 +# 大于 > : 范围命中，一般用在数字范围里面，比如 qid>100000，是说qid范围大于100000都命中 +# 小于 < : 范围命中，类似大于，不过是小于的操作，比如 qid<10000，是说qid小于10000的都命中 +# 包含 IN : 主要是会有一个词表，如果用户命中这个词表，说明包含在这个词表内算是命中。语句类似：qid-IN-xxx_list +# 不包含 NOTIN : 也是有一个词表，不过这句是说如果用户没有命中这个词表，说明通过这个规则。语句类似：qid-NOTIN-xxx_list +# +# +# (注意，只有整形和IP地址类型才支持范围，字符串必须是在一个词表文件才行) +# +# 第三组[]： 配置是代表我们操作的判断条件，格式是固定的。 +# +# 比如 [time=86400; count=50;] [result=2; return=401] +# base : 一般配置问[base]类型的rule才需要本条件，这是一个基础条件，就是说满足了base，然后才回去进行后面的这些条件判断。应用场合类似：提问操作超过30次以后，用户每隔2秒才能提交。如果本选项值是0或者没有配置，则按照普通操作进行。 +# time : 是时间，单位是秒 +# count : 是本规则上限的操作次数，0是代表无限，正整数是操作次数。 +# +# 第四组[]：配置的是满足我们判断条件的返回结果，格式是固定的 +# result : 是说返回我们在[result]中配置的哪个结果 +# return : 是说当前这个规则返回给调用方的一个规则号码结果整数，这个数据会压在我们 result 里返回给调用方 +# +# #示例：提问，同一qid一天超过10次后出验证码 +# rule : [count] [act=add_ask;qid=+;] [time=86400; count=10;] [result=3; return=208] +# +# #示例：提问，同IP，超过30次后，每次提问间隔2秒 +# rule : [base] [act=add_ask;ip=+;] [base=30; time=2; count=1;] [result=2; return=204] +# +# +# +# [result]返回结果配置： +# 返回结果配置比较简单，主要也是分为2块。 +# 冒号左侧的 数字 是代表这是一条返回结果配置，冒号右侧是我们返回给前端的json格式数据。 +# 右侧json格式数据是固定的，类似格式： +# 0 : { "ret_type":0, "ret_code" : 0, "err_no":0, "err_msg":"", "str_reason":"Allow", "need_vcode":0, "vcode_len":4, "vcode_type":0, "other":"", "version":0 } +# +# 字段描述： +# ret_type : 是result 的数字 +# ret_code : 是规则最后配置 return 的整数，前端可以在获取后做一些提示操作，或者记录操作 +# err_no : 错误号(一般有错误的情况) +# err_msg : 错误信息(一般有特殊错误信息) +# str_reason : 操作结果信息(参考使用) +# need_vcode : 是否出验证码，0不出，1出 +# vcode_len : 验证码长度，一般是4位(预留字段，需要看验证码是否支持) +# vode_type : 验证码类型，比如是普通验证码、中文验证码、语音验证码、九宫格验证码等 +# other : 预留字段 +# version : 当前返回信息的版本，预留字段 +# +# +# +# +###################################### 配置说明结束线 ######################################## \ No newline at end of file diff --git a/doc/README.md b/doc/README.md index 769fd7f..970214a 100644 --- a/doc/README.md +++ b/doc/README.md @@ -22,7 +22,7 @@ package main import ( "errors" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "sort" "strconv" "strings" @@ -52,7 +52,7 @@ type KoalaRule struct { /** * 浏览；count规则缓存查询、比较 */ -func (this *KoalaRule) countBrowse(cacheKey string) (bool, error) { +func (k *KoalaRule) countBrowse(cacheKey string) (bool, error) { redisConn := RedisPool.Get() defer redisConn.Close() @@ -67,7 +67,7 @@ func (this *KoalaRule) countBrowse(cacheKey string) (bool, error) { return false, err } //println(cacheKey, " --> value:", cacheValue) - if this.count == 0 || this.count > int32(cacheValue) { + if k.count == 0 || k.count > int32(cacheValue) { return false, nil } return true, nil @@ -76,7 +76,7 @@ func (this *KoalaRule) countBrowse(cacheKey string) (bool, error) { /** * 更新；count规则缓存更新 */ -func (this *KoalaRule) countUpdate(cacheKey string) error { +func (k *KoalaRule) countUpdate(cacheKey string) error { redisConn := RedisPool.Get() defer redisConn.Close() @@ -87,18 +87,18 @@ func (this *KoalaRule) countUpdate(cacheKey string) error { } // set new cache if exists == 0 { - var expiretime int32 + var expireTime int32 // 86400 按照 自然天计算 过期时间 - if this.time == 86400 { + if k.time == 86400 { y, m, d := time.Now().Date() loc, _ := time.LoadLocation("Asia/Shanghai") dayEnd := time.Date(y, m, d, 23, 59, 59, 0, loc).Unix() - expiretime = int32(dayEnd - time.Now().Unix()) + expireTime = int32(dayEnd - time.Now().Unix()) } else { - expiretime = this.time + expireTime = k.time } - if _, err := redis.String(redisConn.Do("SETEX", cacheKey, expiretime, 1)); err != nil { + if _, err := redis.String(redisConn.Do("SETEX", cacheKey, expireTime, 1)); err != nil { return err } @@ -149,7 +149,7 @@ func (this *KoalaRule) countUpdate(cacheKey string) error { * leak模式--查询 * */ -func (this *KoalaRule) leakBrowse(cacheKey string) (bool, error) { +func (k *KoalaRule) leakBrowse(cacheKey string) (bool, error) { redisConn := RedisPool.Get() defer redisConn.Close() @@ -159,7 +159,7 @@ func (this *KoalaRule) leakBrowse(cacheKey string) (bool, error) { if listLen, err = redis.Int(redisConn.Do("LLEN", cacheKey)); err != nil { return false, err } - if listLen == 0 || listLen <= int(this.count) { + if listLen == 0 || listLen <= int(k.count) { return false, nil } @@ -168,10 +168,10 @@ func (this *KoalaRule) leakBrowse(cacheKey string) (bool, error) { }() now := time.Now().Unix() - if edgeElement, err = redis.Int64(redisConn.Do("LINDEX", cacheKey, this.count)); err != nil { + if edgeElement, err = redis.Int64(redisConn.Do("LINDEX", cacheKey, k.count)); err != nil { return false, err } - if int32(now-edgeElement) <= this.time { + if int32(now-edgeElement) <= k.time { return true, nil } return false, nil @@ -181,11 +181,11 @@ func (this *KoalaRule) leakBrowse(cacheKey string) (bool, error) { * leak模式--清理 * 清理队尾过期多余元素 */ -func (this *KoalaRule) leakClear(cacheKey string, listLen int) { +func (k *KoalaRule) leakClear(cacheKey string, listLen int) { redisConn := RedisPool.Get() defer redisConn.Close() - for listLen > int(this.count+1) { + for listLen > int(k.count+1) { if _, err := redis.Int64(redisConn.Do("RPOP", cacheKey)); err != nil { return } @@ -196,7 +196,7 @@ func (this *KoalaRule) leakClear(cacheKey string, listLen int) { /** * leak模式--更新 */ -func (this *KoalaRule) leakUpdate(cacheKey string) error { +func (k *KoalaRule) leakUpdate(cacheKey string) error { redisConn := RedisPool.Get() defer redisConn.Close() @@ -214,7 +214,7 @@ func (this *KoalaRule) leakUpdate(cacheKey string) error { * leak模式--反馈 * 根据指令，减少桶内若干元素 */ -func (this *KoalaRule) leakFeedback(cacheKey string, feedback int) error { +func (k *KoalaRule) leakFeedback(cacheKey string, feedback int) error { redisConn := RedisPool.Get() defer redisConn.Close() diff --git "a/doc/koala_\350\247\204\345\210\231\350\257\264\346\230\216.txt" "b/doc/koala_\350\247\204\345\210\231\350\257\264\346\230\216.txt" index 476a356..47ce7c2 100644 --- "a/doc/koala_\350\247\204\345\210\231\350\257\264\346\230\216.txt" +++ "b/doc/koala_\350\247\204\345\210\231\350\257\264\346\230\216.txt" @@ -1,179 +1,178 @@ -###################### -# �ʱ��ļ� -###################### -[dicts] -global_qid_whitelist : etc/global_qid_whitelist.dat -global_qid_blacklist : etc/global_qid_blacklist.dat - -global_ip_whitelist : etc/global_ip_whitelist.dat -global_ip_blacklist : etc/global_ip_blacklist.dat -answer_qid_blacklist: etc/answer_qid_blacklist.dat - -###################### -# �����б� -###################### -[rules] - -�������������ɸ�key=value�������Զ��塣 -result����ʾԤ��ķ������ͣ��������еġ����ؽ�����á������е�ij�ж�Ӧ�����й���ʱ�����ش��е����ݡ� -return��Ψһ��ʶһ����������ʱ����д�ڷ�������� - -ֱ��������direct����ʾ��������������ƥ�䣬����Ҫ������ֱ�ӷ���Ԥ������������ں����������� - #ȫ�ֺ����� - rule : [direct] [qid @ global_qid_blacklist] [time=1; count=0;] [result=2; return=103] - -������count����ʾ��timeʱ�䣨�룩�� <= count�Ρ� -��Ҫ��ֵ��time��count -@ ��+���Ŵ���key��value�����Ƿǿյ�����ֵ�� - #ÿ�죬����qid�û������ʲ������ܳ���1000�� - rule : [count] [act=ask;qid=+] [time=86400; count=1000;] [result=2; return=111] - -@value�ı���ʽ��֧�� >,<,=,�Լ�!=(�ǵ���)�� - #ÿ�죬qid���ִ���200000000���ˣ�ÿ������������ֹ����10 - rule : [count] [act=ask;qid>200000000] [time=86400; count=10;] [result=2; return=112] - -@ value�ı���ʽ��֧��ip�εļ��ֱ��������ͨ��������������롣 - #�ش�ָ��ip���䣬�ش𳬹�50���Ժ󣬳�4λ��֤�� - rule : [count] [act=answer;ip=192.168.0.1-192.168.0.255,10.0.24.*] [time=86400; count=50;] [result=3; return=113] - -������base������һ��base�������������������������ϡ�����ʾ������һ�������ﵽ��ֵ���ڶ���������ʼ��Ч�� -��һ��������ʱ���������ģ���һ�죩����base���ǵ�һ������������ֵ��ʣ�µ�time��count���������ڶ��������� -��Ҫ�ķ�ֵ��base��time��count - #���ʣ�ͬһIP��ÿ�쳬��10�κ�ÿ�����ʼ��5�� - rule : [base] [act=ask;ip=+;] [base=10; time=5; count=1;] [result=2; return=115] - -©Ͱ��leak��������������count�����ͣ����ǻ��¼ÿ�η��ʵ�ʱ�䣬������Ϊ�������ڣ��������з��ʼ�¼ͬʱ��ʧ�� -��Ҫ�ķ�ֵ��time��count -��ѡ��ֵ��erase1��erase2 -erase1/erase2��������Ϊ������ͨ�����õ����ķ����ӿ�ʵ�� - #����indexҳ��ͬһip��100���ڲ�����10�Σ� - rule : [leak] [act=index_page;ip=+] [time=100; count=10; erase1=2; erase2=4] [result=2; return=1201] - -####################### -# ���ؽ������ -####################### -[result] - -#Ĭ�Ϲ��� (Ĭ����ͨ��״̬) -0 : { "Ret_type":0, "Ret_code" : 0, "Err_no":0, "Err_msg":"", "Str_reason":"Allow", "Need_vcode":0, "Vcode_len":0, "Vcode_type":0, "Other":"", "Version":0 } - -#ͨ�� (û�������κο��ƣ�ͨ�ù���) -1 : { "Ret_type":1, "Ret_code" : 0, "Err_no":0, "Err_msg":"", "Str_reason":"Allow", "Need_vcode":0, "Vcode_len":0, "Vcode_type":0, "Other":"", "Version":0 } - -#��ͨ�� (������Ƶ�ʿ��ƣ�������֤�룬ֱ�ӽ�ֹ����) -2 : { "Ret_type":2, "Ret_code" : 0, "Err_no":10, "Err_msg":"", "Str_reason":"Deny", "Need_vcode":0, "Vcode_len":0, "Vcode_type":0, "Other":"", "Version":0 } - -#��4λ��֤�� -3 : { "Ret_type":3, "Ret_code" : 0, "Err_no":20, "Err_msg":"", "Str_reason":"Vcode", "Need_vcode":1, "Vcode_len":4, "Vcode_type":0, "Other":"", "Version":0 } - -#��6λ��֤�� -4 : { "Ret_type":4, "Ret_code" : 0, "Err_no":21, "Err_msg":"", "Str_reason":"Vcode", "Need_vcode":1, "Vcode_len":6, "Vcode_type":0, "Other":"", "Version":0 } - - -###################################### ����˵���ָ��� ######################################## -# -# ��Koala �����ļ�ʹ��˵���� -# -# ������Ҫ��Ϊ�������֣� -# -# [dicts] : �ʱ����� -# [rules] : �������� -# [result]: ���ؽ������ -# -# -# [dicts]�ʱ����ã� -# -# ��Ҫ����ͨ��ð�� : ���ֵ��������࣬�����������ļ��ĵ����֣���Ҫ���ṩ��rule����ʹ�õģ��ļ����ݱ�����һ��һ�������ݣ���Ҫ�Ǹ�ij���������бȶԣ��Ҳ���ֵ����ļ�·������Ҫ������ʱ��ļ�ʵ���ڵ�ǰ��������·���� -# ʾ���� -# global_qid_whitelist : /home/q/koala/data/global_qid_whitelist.dat -# ��˵һ������ global_qid_whitelist �Ĵʱ��ļ���·���� /home/q/koala/data/global_qid_whitelist.dat -# -# -# [rules]�������ã� -# -# !!ע�⣺�������������Ǵ��ϵ��£�����ǰ��Ĺ������ȼ����ں���Ĺ������ǰ��������о�ֱ�ӷ��أ����˼�����Ĺ���!! -# -# rule���ñȽϸ��ӣ���ҪҲ�Ƿ�Ϊ���顣 -# ð������ rule �Ǵ�������һ���������ã�ð���Ҳ���4������ [] �������ֵĵ����ݼ��� -# ��һ��[]�� ��Ҫ�DZ�ʾ����������ʲô���͵ģ�Ŀǰ֧�� [direct] [count] �� [base] �������� -# [direct] ���Dz���Ҫ�κ�cache����ֵ�Ĺ���ƥ���˾�ֱ�ӷ��� -# [count] ���;���һ���׼�ļ������� -# [base] �������������ͣ�һ���������ơ����ʲ�������30���Ժ��û�ÿ��2������ύ�����ֿ��ƣ������ڵ������ֶ��л���Ҫ����base�ֶ� -# �ڶ���[]���м���ǿͻ��˴��ݲ������ϣ������ṹ�ǣ�key=value; ����ʽ��ÿ�������м�ʹ�÷ֺ����֡����value��ȡֵ�������¼��֣� -# ����ȡֵ�� -# ���� act=answer������˵���key��act�ģ�ֵ�ǹ̶���answer�������б�key���������� -# ���� act!=answer, ����˵ֻҪ�������������act�ֶΣ����Ҳ����� answer �������� -# -# ��Χֵ����Χֵ���� ���䡢�����и�ĵ����������ݣ� -# qid=10,23,45 : ����˵���qid������3�����ֲ������У���֧���ַ������ͣ��ַ���������Ҫ���ڴʱ��ļ��� -# qid=1-2147483647 : ����˵qid��ȡֵ��Χ�� 1 ~ 2147483647 ���ж��� -# qid>10000 : ��˵qid����10000�����з�Χ���� -# ip=192.168.0.* : ��˵ǰ�����ε�ַһ�¾������У���ͬ�ڣ�192.168.0.1-192.168.0.256��Ҳ����ʹ�� 192.168.*.* ���ָ���Χ�� -# ip=192.168.0.1-192.168.0.255 : ����һ��IP��ַ�ķ�Χֵ���������ip��ַ��Χ��ip��ַ��Χ��������*��ͨ�� -# qid @ global_qid_whitelist : ���������˵�������õ��ļ��б���ֻҪ��ǰqid���������У��������У���ͬ�� qid=10,23,45 �������ã�ֻ�ǿ������Ӻܶ� -# ip=192.168.0.1-192.168.0.256,10.0.24.* ��˵�����η�Χ�ڵ�ip�����б����� -# -# -# ���������ͣ� -# ���� # : �������������DZ�ע�͵ģ�һ����������� -# ������ [] : ��Ҫ����Ϊij��β����ı�ʾ����[rule]���̶ֹ���group�����Ų�ͬ -# �ֺ� ; : ������������ֱ�ӽ����� -# �Ⱥ� = : ����key��value�ṹ�ĵ��ڸ�ֵ�IJ��� -# ������ != : �������������value�����������key���Ǹ��Ƚϲ��� -# ���� , : �����ж����ֵ���߷�Χֵ����Ҫʹ�ö���һһ��������һ�����򡱲��� -# �Ӻ� + : �����������ֶΰ���������ֵ��ʲô����Ҫ������һ��Ҫ������ֵ -# �Ǻ� * : ��Ҫ����IP��ַ����������ͨ����IJ��� -# ð�� : : ��Ҫ���ڸ�ֵ����ֶ����ֵ���; -# ���� {~} : ��һ���������������Ҫ������value���棬��� act=answer,ask{*} ����һ���������ǰ������������ϲ���һ�����������ǽ���ͳһ�������������� act=answer,ask �����Ƿֳ����������ģ�ʹ��{~} ���ܹ��ϲ���һ�������ˣ����ijЩ������Ҫ�ļ������� -# ���� - : Ŀǰ��Ҫ�����ڷ�Χ��������� 1-100����˵1��100����������У������� qid-IN-xxx_list Ҳ��˵qid��xxx_list��ʱ������� -# ���� > : ��Χ���У�һ���������ַ�Χ���棬���� qid>100000����˵qid��Χ����100000������ -# С�� < : ��Χ���У����ƴ��ڣ�������С�ڵIJ��������� qid<10000����˵qidС��10000�Ķ����� -# ���� @ : ��Ҫ�ǻ���һ���ʱ�������û���������ʱ���˵������������ʱ����������С�������ƣ�qid-IN-xxx_list -# ������ !@ : Ҳ����һ���ʱ������������˵����û�û����������ʱ���˵��ͨ���������������ƣ�qid-NOTIN-xxx_list -# -# -# (ע�⣬ֻ�����κ�IP��ַ���Ͳ�֧�ַ�Χ���ַ�����������һ���ʱ��ļ�����) -# -# ������[]�� �����Ǵ������Dz������ж���������ʽ�ǹ̶��ġ� -# -# ���� [time=86400; count=50;] [result=2; return=401] -# base : һ��������[base]���͵�rule����Ҫ������������һ����������������˵������base��Ȼ��Ż�ȥ���к������Щ�����жϡ�Ӧ�ó������ƣ����ʲ�������30���Ժ��û�ÿ��2������ύ�������ѡ��ֵ��0����û�����ã�������ͨ�������С� -# time : ��ʱ�䣬��λ���� -# count : �DZ��������޵IJ���������0�Ǵ������ޣ��������Dz��������� -# -# ������[]�����õ������������ж������ķ��ؽ������ʽ�ǹ̶��� -# result : ��˵����������[result]�����õ��ĸ���� -# return : ��˵��ǰ������򷵻ظ����÷���һ�����������������������ݻ�ѹ������ result �ﷵ�ظ����÷� -# -# #ʾ�������ʣ�ͬһqidһ�쳬��10�κ����֤�� -# rule : [count] [act=ask;qid=+;] [time=86400; count=10;] [result=3; return=208] -# -# #ʾ�������ʣ�ͬIP������30�κ�ÿ�����ʼ��2�� -# rule : [base] [act=ask;ip=+;] [base=30; time=2; count=1;] [result=2; return=204] -# -# -# -# [result]���ؽ�����ã� -# -# ���ؽ�����ñȽϼ򵥣���ҪҲ�Ƿ�Ϊ2�顣 -# ð������ ���� �Ǵ�������һ�����ؽ�����ã�ð���Ҳ������Ƿ��ظ�ǰ�˵�json��ʽ���ݡ� -# �Ҳ�json��ʽ�����ǹ̶��ģ����Ƹ�ʽ����ע�⣺ÿ���ֶε�����ĸ�����Ǵ�д����ȻGo�ᱨ������ -# 0 : { "Ret_type":0, "Ret_code" : 0, "Err_no":0, "Err_msg":"", "Str_reason":"Allow", "Need_vcode":0, "Vcode_len":4, "Vcode_type":0, "Other":"", "Version":0 } -# -# �ֶ������� -# ret_type : ��result ������ -# ret_code : �ǹ���������� return ��������ǰ�˿����ڻ�ȡ����һЩ��ʾ���������߼�¼���� -# err_no : �����(һ���д�������) -# err_msg : ������Ϣ(һ�������������Ϣ) -# str_reason : ���������Ϣ(�ο�ʹ��) -# need_vcode : �Ƿ����֤�룬0������1�� -# vcode_len : ��֤�볤�ȣ�һ����4λ(Ԥ���ֶΣ���Ҫ����֤���Ƿ�֧��) -# vode_type : ��֤�����ͣ���������ͨ��֤�롢������֤�롢������֤�롢�Ź�����֤��� -# other : Ԥ���ֶ� -# version : ��ǰ������Ϣ�İ汾��Ԥ���ֶ� -# -# -# -# -###################################### ����˵�������� ######################################## - +###################### +# 词表文件 +###################### +[dicts] +global_qid_whitelist : etc/global_qid_whitelist.dat +global_qid_blacklist : etc/global_qid_blacklist.dat + +global_ip_whitelist : etc/global_ip_whitelist.dat +global_ip_blacklist : etc/global_ip_blacklist.dat +answer_qid_blacklist: etc/answer_qid_blacklist.dat + +###################### +# 规则列表 +###################### +[rules] + +条件区域，用若干个key=value的条件对定义。 +result：表示预设的返回类型，与下文中的“返回结果配置”数组中的某行对应，命中规则时，返回此行的内容。 +return：唯一标识一个规则，命中时，会写在返回数据里。 + +直接命中型direct，表示：若参数与条件匹配，则不需要计数，直接返回预定结果（可用于黑名单等需求） + #全局黑名单 + rule : [direct] [qid @ global_qid_blacklist] [time=1; count=0;] [result=2; return=103] + +计数型count，表示在time时间（秒）内 <= count次。 +必要阀值：time、count +@ “+”号代表key的value可以是非空的任意值。 + #每天，单个qid用户，提问操作不能超过1000次 + rule : [count] [act=ask;qid=+] [time=86400; count=1000;] [result=2; return=111] + +@value的表达式，支持 >,<,=,以及!=(非等于)。 + #每天，qid数字大于200000000的人，每天提问数量禁止超过10 + rule : [count] [act=ask;qid>200000000] [time=86400; count=10;] [result=2; return=112] + +@ value的表达式，支持ip段的几种表达，包括“通配符”、子网掩码。 + #回答，指定ip区间，回答超过50次以后，出4位验证码 + rule : [count] [act=answer;ip=192.168.0.1-192.168.0.255,10.0.24.*] [time=86400; count=50;] [result=3; return=113] + +基数型base，增加一个base参数，代表“两个计数的联合”，表示：当第一个计数达到阀值，第二个计数开始生效。 +第一个计数的时间是隐含的（是一天），而base就是第一个计数的数量值。剩下的time、count用来代表第二个计数。 +必要的阀值：base、time、count + #提问，同一IP，每天超过10次后，每次提问间隔5秒 + rule : [base] [act=ask;ip=+;] [base=10; time=5; count=1;] [result=2; return=115] + +漏桶型leak，基本作用类似count计数型，但是会记录每次访问的时间，不会因为计数过期，导致所有访问记录同时消失。 +必要的阀值：time、count +可选阀值：erase1、erase2 +erase1/erase2：用于行为反馈，通过调用单独的反馈接口实现 + #访问index页，同一ip，100秒内不超过10次， + rule : [leak] [act=index_page;ip=+] [time=100; count=10; erase1=2; erase2=4] [result=2; return=1201] + +####################### +# 返回结果配置 +####################### +[result] + +#默认规则 (默认是通过状态) +0 : { "Ret_type":0, "Ret_code" : 0, "Err_no":0, "Err_msg":"", "Str_reason":"Allow", "Need_vcode":0, "Vcode_len":0, "Vcode_type":0, "Other":"", "Version":0 } + +#通过 (没有命中任何控制，通用规则) +1 : { "Ret_type":1, "Ret_code" : 0, "Err_no":0, "Err_msg":"", "Str_reason":"Allow", "Need_vcode":0, "Vcode_len":0, "Vcode_type":0, "Other":"", "Version":0 } + +#不通过 (命中了频率控制，不出验证码，直接禁止操作) +2 : { "Ret_type":2, "Ret_code" : 0, "Err_no":10, "Err_msg":"", "Str_reason":"Deny", "Need_vcode":0, "Vcode_len":0, "Vcode_type":0, "Other":"", "Version":0 } + +#出4位验证码 +3 : { "Ret_type":3, "Ret_code" : 0, "Err_no":20, "Err_msg":"", "Str_reason":"Vcode", "Need_vcode":1, "Vcode_len":4, "Vcode_type":0, "Other":"", "Version":0 } + +#出6位验证码 +4 : { "Ret_type":4, "Ret_code" : 0, "Err_no":21, "Err_msg":"", "Str_reason":"Vcode", "Need_vcode":1, "Vcode_len":6, "Vcode_type":0, "Other":"", "Version":0 } + + +###################################### 配置说明分割线 ######################################## +# +# 【Koala 配置文件使用说明】 +# +# 配置主要分为三个部分： +# +# [dicts] : 词表配置 +# [rules] : 规则配置 +# [result]: 返回结果配置 +# +# +# [dicts]词表配置： +# +# 主要就是通过冒号 : 区分的左右两侧，左侧的是数据文件的的名字，主要是提供给rule规则使用的，文件内容必须是一行一条的内容，主要是跟某个参数进行比对，右侧出现的是文件路径，主要是这个词表文件实际在当前服务器的路径。 +# 示例： +# global_qid_whitelist : /home/q/koala/data/global_qid_whitelist.dat +# 是说一个叫做 global_qid_whitelist 的词表文件，路径是 /home/q/koala/data/global_qid_whitelist.dat +# +# +# [rules]规则配置： +# +# !!注意：整个规则运行是从上到下，排在前面的规则优先级高于后面的规则，如果前面规则命中就直接返回，不顾及后面的规则。!! +# +# rule配置比较复杂，主要也是分为三块。 +# 冒号左侧的 rule 是代表这是一条规则配置，冒号右侧是4组依赖 [] 进行区分的的数据集。 +# 第一组[]： 主要是表示这条规则是什么类型的，目前支持 [direct] [count] 和 [base] 三种类型 +# [direct] 就是不需要任何cache计数值的规则，匹配了就直接返回 +# [count] 类型就是一般标准的计数类型 +# [base] 这种是特殊类型，一般满足类似“提问操作超过30次以后，用户每隔2秒才能提交”这种控制，另外在第三组字段中还需要增加base字段 +# 第二组[]：中间的是客户端传递参数集合，基本结构是：key=value; 的形式，每个参数中间使用分号区分。这个value的取值包括如下几种： +# 绝对取值： +# 比如 act=answer，就是说这个key是act的，值是固定是answer才算命中本key，否则不命中 +# 比如 act!=answer, 就是说只要这个操作包含了act字段，并且不等于 answer 都算命中 +# +# 范围值：范围值包括 区间、逗号切割的单独多条数据， +# qid=10,23,45 : 就是说这个qid包含这3个数字才算命中，不支持字符串类型，字符串类型需要放在词表文件中 +# qid=1-2147483647 : 就是说qid的取值范围从 1 ~ 2147483647 命中都算 +# qid>10000 : 是说qid大于10000的所有范围命中 +# ip=192.168.0.* : 是说前面三段地址一致就算命中，等同于：192.168.0.1-192.168.0.256，也可以使用 192.168.*.* 这种更大范围的 +# ip=192.168.0.1-192.168.0.255 : 这是一个IP地址的范围值，代表这个ip地址范围，ip地址范围还可以用*来通配 +# qid @ global_qid_whitelist : 这个配置是说上面配置的文件列表，只要当前qid包含在其中，就算命中，等同于 qid=10,23,45 这种配置，只是可以增加很多 +# ip=192.168.0.1-192.168.0.256,10.0.24.* 是说这两段范围内的ip都命中本操作 +# +# +# 操作符类型： +# 井号 # : 代表本行内容是被注释的，一般出现在行首 +# 中括号 [] : 主要是作为某大段操作的表示，跟[rule]这种固定的group中括号不同 +# 分号 ; : 代表两个参数直接结束了 +# 等号 = : 代表key、value结构的等于赋值的操作 +# 不等于 != : 代表不满足这个value就是命中这个key，是个比较操作 +# 逗号 , : 代表有多个数值或者范围值，需要使用逗号一一描述，是一个“或”操作 +# 加号 + : 代表本参数字段包含，具体值是什么不重要，就是一定要包含本值 +# 星号 * : 主要是在IP地址等数据里面通配符的操作 +# 冒号 : : 主要是在赋值大块字段区分的用途 +# 特殊 {~} : 这一个特殊操作符，主要是用在value后面，针对 act=answer,ask{*} 这样一个操作，是把这两个操作合并成一个计数，我们叫做统一计数操作，对于 act=answer,ask 我们是分成两个计数的，使用{~} 就能够合并成一个计数了，解决某些特殊需要的计数场合 +# 减号 - : 目前主要是用在范围操作里，比如 1-100，是说1到100这个区间命中，或者是 qid-IN-xxx_list 也是说qid在xxx_list这词表中命中 +# 大于 > : 范围命中，一般用在数字范围里面，比如 qid>100000，是说qid范围大于100000都命中 +# 小于 < : 范围命中，类似大于，不过是小于的操作，比如 qid<10000，是说qid小于10000的都命中 +# 包含 @ : 主要是会有一个词表，如果用户命中这个词表，说明包含在这个词表内算是命中。语句类似：qid-IN-xxx_list +# 不包含 !@ : 也是有一个词表，不过这句是说如果用户没有命中这个词表，说明通过这个规则。语句类似：qid-NOTIN-xxx_list +# +# +# (注意，只有整形和IP地址类型才支持范围，字符串必须是在一个词表文件才行) +# +# 第三组[]： 配置是代表我们操作的判断条件，格式是固定的。 +# +# 比如 [time=86400; count=50;] [result=2; return=401] +# base : 一般配置问[base]类型的rule才需要本条件，这是一个基础条件，就是说满足了base，然后才回去进行后面的这些条件判断。应用场合类似：提问操作超过30次以后，用户每隔2秒才能提交。如果本选项值是0或者没有配置，则按照普通操作进行。 +# time : 是时间，单位是秒 +# count : 是本规则上限的操作次数，0是代表无限，正整数是操作次数。 +# +# 第四组[]：配置的是满足我们判断条件的返回结果，格式是固定的 +# result : 是说返回我们在[result]中配置的哪个结果 +# return : 是说当前这个规则返回给调用方的一个规则号码结果整数，这个数据会压在我们 result 里返回给调用方 +# +# #示例：提问，同一qid一天超过10次后出验证码 +# rule : [count] [act=ask;qid=+;] [time=86400; count=10;] [result=3; return=208] +# +# #示例：提问，同IP，超过30次后，每次提问间隔2秒 +# rule : [base] [act=ask;ip=+;] [base=30; time=2; count=1;] [result=2; return=204] +# +# +# +# [result]返回结果配置： +# +# 返回结果配置比较简单，主要也是分为2块。 +# 冒号左侧的 数字 是代表这是一条返回结果配置，冒号右侧是我们返回给前端的json格式数据。 +# 右侧json格式数据是固定的，类似格式：（注意：每个字段的首字母必须是大写，不然Go会报错！） +# 0 : { "Ret_type":0, "Ret_code" : 0, "Err_no":0, "Err_msg":"", "Str_reason":"Allow", "Need_vcode":0, "Vcode_len":4, "Vcode_type":0, "Other":"", "Version":0 } +# +# 字段描述： +# ret_type : 是result 的数字 +# ret_code : 是规则最后配置 return 的整数，前端可以在获取后做一些提示操作，或者记录操作 +# err_no : 错误号(一般有错误的情况) +# err_msg : 错误信息(一般有特殊错误信息) +# str_reason : 操作结果信息(参考使用) +# need_vcode : 是否出验证码，0不出，1出 +# vcode_len : 验证码长度，一般是4位(预留字段，需要看验证码是否支持) +# vode_type : 验证码类型，比如是普通验证码、中文验证码、语音验证码、九宫格验证码等 +# other : 预留字段 +# version : 当前返回信息的版本，预留字段 +# +# +# +# +###################################### 配置说明结束线 ######################################## \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..af4390f --- /dev/null +++ b/go.mod @@ -0,0 +1,8 @@ +module github.com/heiyeluren/koala + +go 1.16 + +require ( + github.com/gomodule/redigo v1.8.5 + gopkg.in/yaml.v2 v2.3.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..233a646 --- /dev/null +++ b/go.sum @@ -0,0 +1,13 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gomodule/redigo v1.8.5 h1:nRAxCa+SVsyjSBrtZmG/cqb6VbTmuRzpg/PoTFlpumc= +github.com/gomodule/redigo v1.8.5/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/koala/KoalaKey.go b/koala/KoalaKey.go new file mode 100644 index 0000000..c474ab1 --- /dev/null +++ b/koala/KoalaKey.go @@ -0,0 +1,268 @@ +/** + * Koala Rule Engine Core + * + * @package: main + * @desc: koala engine code + * + * @author: heiyeluren + * @github: https://github.com/heiyeluren + * @blog: https://blog.csdn.net/heiyeshuwu + * + */ + +package koala + +import ( + "errors" + "strconv" + "strings" +) + +// KoalaKey interface ；用于支持多种 key 类型 +type KoalaKey interface { + // 从配置字符串识别并生成 key 结构 + build(op, k, v string) error + + // 匹配 s 是否包含于 key 的范围 + matches(s string) bool + + // 仅调试用途 + dump() string +} + +// KoalaKey 有两个子类型 +// 集合 GroupKey +// 范围 RangeKey + +// GroupKey 集合 key 类型；满足 KoalaKey interface +type GroupKey struct { + set map[string]string + inverse bool // 取反标记；@,!@ + combine bool // 合并标记；{~} +} + +/** + * dump() + */ +func (g *GroupKey) dump() string { + ret := "" + if g.inverse { + ret += "!" + } + if g.combine { + ret += "~" + } + for v := range g.set { + ret += v + "," + } + return ret +} + +/** + * build() + */ +func (g *GroupKey) build(sp, k, v string) error { + // 词表识别 + g.set = make(map[string]string, 10) + g.combine = false + g.inverse = strings.HasSuffix(k, "!") + var isPresent bool + v = strings.Trim(v, emptyRunes) + if sp == "@" { + g.set, isPresent = TempPolicy.dictsTable[v] + if !isPresent { + return errors.New("rule build error: Dict not present") + } + return nil + } + if sp == "=" { + g.combine = strings.HasSuffix(v, "{~}") + v = strings.Trim(v, "{~}") + elements := strings.Split(v, ",") + for _, e := range elements { + item := strings.Trim(e, emptyRunes) + g.set[item] = item + } + } + return nil +} + +/** + * matches() + */ +func (g *GroupKey) matches(s string) bool { + s = strings.Trim(s, emptyRunes) + if _, OK := g.set[s]; OK != g.inverse { + return true + } + return false +} + +// RangeKey 范围key；一个范围key可以包含多个范围区间 +type RangeKey struct { + scopes []*Scope + inverse bool // 取反标记 ! +} + +/** + * dump + */ +func (k *RangeKey) dump() string { + ret := "" + if k.inverse { + ret += "!" + } + for _, scop := range k.scopes { + ret += scop.dump() + "&" + } + return ret +} + +/** + * build + */ +func (k *RangeKey) build(sp, ki, v string) error { + // 范围识别 + var err error + var int64Val int64 + k.inverse = strings.HasSuffix(ki, "!") + v = strings.Trim(v, emptyRunes) + if sp == "<" { + oneScope := new(Scope) + oneScope.op = sp + if int64Val, err = strconv.ParseInt(v, 10, 64); err != nil { + return errors.New("rule syntax error: < error,not integer") + } + oneScope.start = int64Val + k.scopes = []*Scope{oneScope} + return nil + } + if sp == ">" { + oneScope := new(Scope) + oneScope.op = sp + if int64Val, err = strconv.ParseInt(v, 10, 64); err != nil { + return errors.New("rule syntax error: > error,not integer") + } + oneScope.end = int64Val + k.scopes = []*Scope{oneScope} + return nil + } + // sp 是 = + if v == "+" { + oneScope := new(Scope) + oneScope.op = v + k.scopes = []*Scope{oneScope} + return nil + } + k.scopes = []*Scope{} + parts := strings.Split(v, ",") + for _, sc := range parts { + oneScope := new(Scope) + if err = oneScope.build(sc); err != nil { + return err + } + k.scopes = append(k.scopes, oneScope) + } + return nil +} + +/** + * matches + */ +func (k *RangeKey) matches(s string) bool { + // + 号，任意值逻辑，直接matche + for _, sco := range k.scopes { + if sco.op == "+" { + return true + } + } + int64val, err := ToInteger64(s) + if err != nil { + return false + } + isIn := false + for _, sco := range k.scopes { + if sco.matches(int64val) { + isIn = true + break + } + } + + return isIn != k.inverse +} + +// Scope 范围 scope 类型，标识一个数值区间，如：>100, 1-9 +type Scope struct { + op string // -,+,>,< + start int64 + end int64 +} + +/** + * dump + */ +func (s *Scope) dump() string { + a := strconv.FormatInt(s.start, 10) + b := strconv.FormatInt(s.end, 10) + return s.op + "^" + a + "^" + b +} + +/** + * build + */ +func (s *Scope) build(sc string) error { + var err error + s.op = "-" + parts := strings.Split(sc, "-") + if len(parts) == 2 { + s.start, err = ToInteger64(strings.Trim(parts[0], emptyRunes)) + if err != nil { + return err + } + s.end, err = ToInteger64(strings.Trim(parts[1], emptyRunes)) + if err != nil { + return err + } + return nil + } + if strings.ContainsAny(sc, "*") && IsIPAddress(sc) { + rep := strings.NewReplacer("*", "0") + addStart := strings.Trim(rep.Replace(sc), emptyRunes) + s.start, err = ToInteger64(addStart) + if err != nil { + return err + } + rep = strings.NewReplacer("*", "255") + addEnd := strings.Trim(rep.Replace(sc), emptyRunes) + s.end, err = ToInteger64(addEnd) + if err != nil { + return err + } + return nil + } + return errors.New("rule syntax error: scope error") +} + +/** + * matches + */ +func (s *Scope) matches(v int64) bool { + switch s.op { + case "+": + return true + case "<": + if v < s.start { + return true + } + case ">": + if v > s.end { + return true + } + case "-": + if v >= s.start && v <= s.end { + return true + } + default: + } + return false +} diff --git a/src/koala/admin.go b/koala/admin.go similarity index 82% rename from src/koala/admin.go rename to koala/admin.go index 3602cd6..c271d27 100644 --- a/src/koala/admin.go +++ b/koala/admin.go @@ -4,25 +4,24 @@ * @package: main * @desc: koala engine - admin web api * - * @author: heiyeluren + * @author: heiyeluren * @github: https://github.com/heiyeluren * @blog: https://blog.csdn.net/heiyeshuwu * */ -package main +package koala /* import ( "fmt" "io/ioutil" "os" - "utility/logger" - "utility/network" + "github.com/heiyeluren/koala/utility" ) 暂时删除 -func (this *FrontServer) DoRuleRewrite(request *network.HttpRequest, response *network.HttpResponse, logHandle *logger.Logger) { +func (s *FrontServer) DoRuleRewrite(request *network.HttpRequest, response *network.HttpResponse, logHandle *logger.Logger) { ext_stream := request.Pstr("rule_stream") if ext_stream == "" { response.Puts(`{"errno": -1, "errmsg": "no rule stream"}`) @@ -46,7 +45,7 @@ func (this *FrontServer) DoRuleRewrite(request *network.HttpRequest, response *n */ /* -func (this *FrontServer) DoDumpCounter(request *network.HttpRequest, response *network.HttpResponse, logHandle *logger.Logger) { +func (s *FrontServer) DoDumpCounter(request *network.HttpRequest, response *network.HttpResponse, logHandle *logger.Logger) { retString := "" rule_no := int32(request.Gint("rule_no")) counters := GetCurrentCounters() diff --git a/koala/cacheSvc.go b/koala/cacheSvc.go new file mode 100644 index 0000000..14231ed --- /dev/null +++ b/koala/cacheSvc.go @@ -0,0 +1,58 @@ +/** + * Koala Rule Engine Core + * + * @package: main + * @desc: koala engine - Redis cache conn pool + * + * @author: heiyeluren + * @github: https://github.com/heiyeluren + * @blog: https://blog.csdn.net/heiyeshuwu + * + */ + +package koala + +import ( + "time" + + "github.com/gomodule/redigo/redis" +) + +// InitRedisPool redis连接池初始化函数 +func InitRedisPool() { + // redis服务 host:port + var server string = Config.Get("redis_server") + // redis服务 口令 + var password string = Config.Get("redis_auth") + // redis连接池 最大空闲连接数 + var maxIdle int = Config.GetInt("redis_pool_maxIdle") + // redis连接池 空闲连接超时时长 + var idleTimeout int = Config.GetInt("redis_pool_idleTimeout") + + var connectTimeout time.Duration = time.Duration(Config.GetInt("externalConnTimeout")) * time.Millisecond + var readTimeout time.Duration = time.Duration(Config.GetInt("externalReadTimeout")) * time.Millisecond + var writeTimeout time.Duration = time.Duration(Config.GetInt("externalWriteTimeout")) * time.Millisecond + + // 新建连接池 + RedisPool = &redis.Pool{ + MaxIdle: maxIdle, + IdleTimeout: time.Duration(idleTimeout) * time.Second, + // dial方法 + Dial: func() (redis.Conn, error) { + c, err := redis.Dial("tcp", server, redis.DialConnectTimeout(connectTimeout), redis.DialReadTimeout(readTimeout), redis.DialReadTimeout(writeTimeout)) + if err != nil { + return nil, err + } + if _, err = c.Do("AUTH", password); err != nil { + c.Close() + return nil, err + } + return c, err + }, + // 连接可用性检查 方法 + TestOnBorrow: func(c redis.Conn, t time.Time) error { + _, err := c.Do("PING") + return err + }, + } +} diff --git a/koala/config.go b/koala/config.go new file mode 100644 index 0000000..dc111ad --- /dev/null +++ b/koala/config.go @@ -0,0 +1,34 @@ +/** + * Koala Rule Engine Core + * + * @package: main + * @desc: koala engine - koala server config parse + * + * @author: heiyeluren + * @github: https://github.com/heiyeluren + * @blog: https://blog.csdn.net/heiyeshuwu + * + */ + +package koala + +import ( + "flag" + + "github.com/heiyeluren/koala/utility" +) + +// NewConfig 启动前加载配置文件 +func NewConfig() *utility.Config { + var F string + flag.StringVar(&F, "f", "", "config file") + flag.Parse() + if F == "" { + panic("usage: ./koala -f etc/koala.conf") + } + config := utility.NewConfig() + if err := config.Load(F); err != nil { + panic(err.Error()) + } + return config +} diff --git a/src/koala/debug.go b/koala/debug.go similarity index 84% rename from src/koala/debug.go rename to koala/debug.go index ccd6ecb..e887a85 100644 --- a/src/koala/debug.go +++ b/koala/debug.go @@ -4,27 +4,26 @@ * @package: main * @desc: koala engine - Test case code * - * @author: heiyeluren + * @author: heiyeluren * @github: https://github.com/heiyeluren * @blog: https://blog.csdn.net/heiyeshuwu * */ -package main +package koala // 测试使用 暂时删除 /* import ( "fmt" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "runtime" "strconv" "time" - "utility/logger" - "utility/network" + "github.com/heiyeluren/koala/utility" ) -func (this *FrontServer) DoLogTest(request *network.HttpRequest, response *network.HttpResponse, logHandle *logger.Logger) { +func (s *FrontServer) DoLogTest(request *network.HttpRequest, response *network.HttpResponse, logHandle *logger.Logger) { logHandle.Debug("[clientip=192.168.0.1 errno=0 errmsg=ok debuginfo]") logHandle.Trace("[clientip=192.168.0.1 errno=0 errmsg=ok traceinfo]") logHandle.Warning("warning test") @@ -34,14 +33,14 @@ func (this *FrontServer) DoLogTest(request *network.HttpRequest, response *netwo response.SetCode(200) } -func (this *FrontServer) DoDumpPolicy(request *network.HttpRequest, response *network.HttpResponse, logHandle *logger.Logger) { +func (s *FrontServer) DoDumpPolicy(request *network.HttpRequest, response *network.HttpResponse, logHandle *logger.Logger) { println(request.GetRemoteIP()) var dumpString string = "" var localPolicy *Policy = GlobalPolicy for _, singleRule := range localPolicy.ruleTable { var singleDump string = "" - singleDump += "methord_ " + singleRule.methord + singleDump += "methord_ " + singleRule.method singleDump += " _keys_ " + dumpkeys(singleRule.keys) singleDump += " _base_ " + strconv.Itoa(int(singleRule.base)) singleDump += " _time_ " + strconv.Itoa(int(singleRule.time)) @@ -62,7 +61,7 @@ func dumpkeys(keys map[string]KoalaKey) string { return ret } -func (this *FrontServer) DoRedisCmd(request *network.HttpRequest, response *network.HttpResponse, logHandle *logger.Logger) { +func (s *FrontServer) DoRedisCmd(request *network.HttpRequest, response *network.HttpResponse, logHandle *logger.Logger) { redisConn := RedisPool.Get() defer redisConn.Close() @@ -86,7 +85,7 @@ func (this *FrontServer) DoRedisCmd(request *network.HttpRequest, response *netw response.SetCode(200) } -func (this *FrontServer) DoRedisMget(request *network.HttpRequest, response *network.HttpResponse, logHandle *logger.Logger) { +func (s *FrontServer) DoRedisMget(request *network.HttpRequest, response *network.HttpResponse, logHandle *logger.Logger) { redisConn := RedisPool.Get() defer redisConn.Close() diff --git a/src/koala/doc.go b/koala/doc.go similarity index 86% rename from src/koala/doc.go rename to koala/doc.go index 1d7ee9d..7c3e705 100644 --- a/src/koala/doc.go +++ b/koala/doc.go @@ -1,33 +1,33 @@ -/** - * Koala Rule Engine Core - * - * @package: main - * @desc: koala engine - Access http api list - * - * @author: heiyeluren - * @github: https://github.com/heiyeluren - * @blog: https://blog.csdn.net/heiyeshuwu - * - */ - -package main - -/* -线上提供接口列表 - -查询接口(命中一条策略结果直接返回) -/rule/browse - -完全查询接口(将所有命中的策略结果返回) -/rule/browse_complete - -多重查询接口 -/multi/browse - -更新接口 -/rule/update - -监控接口 -/monitor/alive - -*/ +/** + * Koala Rule Engine Core + * + * @package: main + * @desc: koala engine - Access http api list + * + * @author: heiyeluren + * @github: https://github.com/heiyeluren + * @blog: https://blog.csdn.net/heiyeshuwu + * + */ + +package koala + +/* +线上提供接口列表 + +查询接口(命中一条策略结果直接返回) +/rule/browse + +完全查询接口(将所有命中的策略结果返回) +/rule/browse_complete + +多重查询接口 +/multi/browse + +更新接口 +/rule/update + +监控接口 +/monitor/alive + +*/ diff --git a/koala/httpFront.go b/koala/httpFront.go new file mode 100644 index 0000000..1efefe4 --- /dev/null +++ b/koala/httpFront.go @@ -0,0 +1,109 @@ +/** + * Koala Rule Engine Core + * + * @package: main + * @desc: koala engine - Http server front api + * + * @author: heiyeluren + * @github: https://github.com/heiyeluren + * @blog: https://blog.csdn.net/heiyeshuwu + * + */ + +package koala + +import ( + "reflect" + "regexp" + "strings" + "time" + + "github.com/heiyeluren/koala/utility" +) + +var frontPattern = regexp.MustCompile("^[a-z][0-9a-z_]*/[a-z][0-9a-z_]*$") + +// FrontServer . +type FrontServer struct{} + +// NewFrontServer . +func NewFrontServer() *FrontServer { + return &FrontServer{} +} + +// FrontListen . +func FrontListen() { + tcpListener, err := utility.TcpListen(Config.Get("listen")) + if err != nil { + panic(err.Error()) + } + for { + tcpConnection, err := tcpListener.Accept() + if err != nil { + continue + } + go FrontDispatch(utility.NewHttpConnection(tcpConnection)) + } +} + +// FrontDispatch . +func FrontDispatch(httpConnection *utility.HttpConnection) { + defer httpConnection.Close() + + request, err := httpConnection.ReadRequest(time.Duration(Config.GetInt("externalReadTimeout")) * time.Millisecond) + if err != nil { + return + } + response := utility.NewHttpResponse() + + // 生成log句柄 + logHandle := utility.NewLogger("") + + pathInfo := strings.Trim(request.PathInfo(), "/") + parts := strings.Split(pathInfo, "/") + if len(parts) == 2 && frontPattern.Match([]byte(pathInfo)) { + methodName := "Do" + strings.Title(parts[0]) + strings.Title(parts[1]) + frontServer := NewFrontServer() + frontServerValue := reflect.ValueOf(frontServer) + methodValue := frontServerValue.MethodByName(methodName) + if methodValue.IsValid() { + requestValue := reflect.ValueOf(request) + responseValue := reflect.ValueOf(response) + logHandleValue := reflect.ValueOf(logHandle) + methodValue.Call([]reflect.Value{requestValue, responseValue, logHandleValue}) + goto finished + } + } + response.SetCode(404) + response.Puts("not found") +finished: + // 每个请求，记录访问日志 + requestLogWrite(request, response, logHandle) + + response.SetHeader("Connection", "close") + httpConnection.WriteResponse(response, time.Duration(Config.GetInt("externalWriteTimeout"))*time.Millisecond) +} + +func requestLogWrite(request *utility.HttpRequest, response *utility.HttpResponse, logHandle *utility.Logger) { + if request.PathInfo() == "/multi/browse" { + // 批量接口，不在此记录notice，在接口内部记录 + return + } + + var logMsg string = "" + logMsg += "[ cip=" + request.GetRemoteIP() + logMsg += " intf=" + request.PathInfo() + + gets := request.Gets() + for k, v := range gets { + logMsg += " " + k + "=" + v + } + posts := request.Posts() + for k, v := range posts { + logMsg += " " + k + "=" + v + } + logMsg += " ]" + logMsg += " [ BodyString=" + response.BodyString() + " ]" + + logHandle.Notice(logMsg) +} diff --git a/koala/koala.go b/koala/koala.go new file mode 100644 index 0000000..c06ddb9 --- /dev/null +++ b/koala/koala.go @@ -0,0 +1,57 @@ +package koala + +import ( + "runtime" + + "github.com/gomodule/redigo/redis" + "github.com/heiyeluren/koala/utility" +) + +// @Project: koala +// @Author: houseme +// @Description: +// @File: koala +// @Version: 1.0.0 +// @Date: 2021/7/13 00:16 +// @Package koala +var ( + // Config koala基本配置 + Config *utility.Config + // RedisPool 全局redis连接池 + RedisPool *redis.Pool + + // PolicyMd5 . + // 全局 md5 值 + // 说明：定期检查 rule配置的变化，与此 md5 比较；实现动态更新规则 + PolicyMd5 string + + // DynamicUpdateFiles . + // 需要检查更新的文件列表 + // 范围：rule 文件 + dicts 文件 + // 赋值：每次reload规则时，将其中的 dicts 文件记录于此 + DynamicUpdateFiles []string +) + +func init() { + // 设置koala进程并发线程数 + runtime.GOMAXPROCS(runtime.NumCPU()) + // 初始化配置 + Config = NewConfig() + // 初始化连接池 + InitRedisPool() +} + +// Run . +func Run() { + // 保存进程的 pid 到文件中，供 stop、restart 脚本引用 + SavePid(Config.Get("pid_file")) + + // 初始化，并启动 logger 协程 + go utility.LogRun(Config.GetAll()) + + // 首次加载规则 + if err := PolicyInterpreter(""); err != nil { + panic(err.Error()) + } + PolicyMd5 = NewPolicyMD5() +} diff --git a/koala/koalaRule.go b/koala/koalaRule.go new file mode 100644 index 0000000..5beace0 --- /dev/null +++ b/koala/koalaRule.go @@ -0,0 +1,537 @@ +/** + * Koala Rule Engine Core + * + * @package: main + * @desc: koala engine - Core Rule Parse engine + * + * @author: heiyeluren + * @github: https://github.com/heiyeluren + * @blog: https://blog.csdn.net/heiyeshuwu + * + */ + +package koala + +import ( + "errors" + "sort" + "strconv" + "strings" + "time" + + "github.com/gomodule/redigo/redis" +) + +const ( + // BaseKeySuffix base附加cache key的后缀；在getCacheKey()的key后追加 + BaseKeySuffix = "_B" +) + +// Rule rule类型 +type Rule struct { + method string //只能为如下四个字符串 count base direct leak + keys map[string]KoalaKey + base int32 + time int32 + count int32 + erase1 int32 + erase2 int32 + result int32 + returnCode int32 +} + +/************************************************************ + KoalaRule 构建，build，相关方法 +************************************************************/ + +// Constructor .KoalaRule 的构造器 +func (k *Rule) Constructor(r string) error { + // [direct] [qid @ global_qid_whitelist] [time=1; count=0;] [result=1; return=101] + // [count] [act=ask;qid=+;] [time=2; count=1;] [result=2; return=201] + // [base] [act=ask;ip=+;] [base=50; time=10; count=1;] [result=2; return=203] + sections := strings.Split(r, "] [") + if len(sections) != 4 { + return errors.New("rule syntax error: section error") + } + for i := range sections { + sections[i] = strings.Trim(sections[i], emptyRunes+"[]") + } + k.method = sections[0] + if k.method != "count" && k.method != "base" && k.method != "direct" && k.method != "leak" { + return errors.New("rule syntax error: method error") + } + k.keys = make(map[string]KoalaKey, 10) + if err := k.getKeys(sections[1]); err != nil { + return err + } + // 解析剩余的 count、returnCode 等 参数 + if err := k.getCountAndRet(sections[2], sections[3]); err != nil { + return err + } + return nil +} + +/** + * 解析 KoalaRule 的 count、returnCode 等 参数 + */ +func (k *Rule) getCountAndRet(val, ret string) error { + val = strings.Trim(val, emptyRunes+";") + ret = strings.Trim(ret, emptyRunes+";") + vals := strings.Split(val, ";") + for _, s := range vals { + s = strings.Trim(s, emptyRunes) + parts := strings.SplitN(s, "=", 2) + if len(parts) != 2 { + return errors.New("rule syntax error: value error") + } + valueName := strings.Trim(parts[0], emptyRunes) + valueData, err := strconv.Atoi(strings.Trim(parts[1], emptyRunes)) + if err != nil { + return errors.New("rule syntax error: value error") + } + switch valueName { + case "base": + k.base = int32(valueData) + case "time": + k.time = int32(valueData) + case "count": + k.count = int32(valueData) + case "erase1": + k.erase1 = int32(valueData) + case "erase2": + k.erase2 = int32(valueData) + default: + return errors.New("rule syntax error: value error") + } + } + rets := strings.Split(ret, ";") + for _, s := range rets { + s = strings.Trim(s, emptyRunes) + parts := strings.SplitN(s, "=", 2) + if len(parts) != 2 { + return errors.New("rule syntax error: value error") + } + valueName := strings.Trim(parts[0], emptyRunes) + valueData, err := strconv.Atoi(strings.Trim(parts[1], emptyRunes)) + if err != nil { + return errors.New("rule syntax error: value error") + } + switch valueName { + case "result": + k.result = int32(valueData) + case "return": + k.returnCode = int32(valueData) + default: + return errors.New("rule syntax error: value error") + } + } + return nil +} + +/** + * 解析 KoalaRule 的 keys 参数 + */ +func (k *Rule) getKeys(ki string) error { + // act=ask;ip=+; + ki = strings.Trim(ki, emptyRunes+";") + allKey := strings.Split(ki, ";") + for i := range allKey { + allKey[i] = strings.Trim(allKey[i], emptyRunes) + + // 抽取 词表 @语句 集合key + // qid @ global_qid_whitelist + parts := strings.SplitN(allKey[i], "@", 2) + if len(parts) == 2 { + keyValue := new(GroupKey) + if err := keyValue.build("@", parts[0], parts[1]); err != nil { + return err + } + keyName := strings.Trim(parts[0], emptyRunes+"!") + k.keys[keyName] = keyValue + continue + } + + // 抽取 小于 < 语句 范围 key + parts = strings.SplitN(allKey[i], "<", 2) + if len(parts) == 2 { + keyValue := new(RangeKey) + if err := keyValue.build("<", parts[0], parts[1]); err != nil { + return err + } + keyName := strings.Trim(parts[0], emptyRunes+"!") + k.keys[keyName] = keyValue + continue + } + + // 抽取 小于 > 语句 范围key + parts = strings.SplitN(allKey[i], ">", 2) + if len(parts) == 2 { + keyValue := new(RangeKey) + if err := keyValue.build(">", parts[0], parts[1]); err != nil { + return err + } + keyName := strings.Trim(parts[0], emptyRunes+"!") + k.keys[keyName] = keyValue + continue + } + + // 处理其他的 以 = 分割的语句，否则报错 + parts = strings.SplitN(allKey[i], "=", 2) + if len(parts) != 2 { + return errors.New("rule syntax error: keys error,miss sp =") + } + if strings.ContainsAny(parts[1], "+-*") { + keyValue := new(RangeKey) // 范围 + if err := keyValue.build("=", parts[0], parts[1]); err != nil { + return err + } + keyName := strings.Trim(parts[0], emptyRunes+"!") + k.keys[keyName] = keyValue + } else { + keyValue := new(GroupKey) // 集合 + if err := keyValue.build("=", parts[0], parts[1]); err != nil { + return err + } + keyName := strings.Trim(parts[0], emptyRunes+"!") + k.keys[keyName] = keyValue + } + + } + return nil +} + +/************************************************************ + KoalaRule 使用过程，matche，相关方法 +************************************************************/ + +/** + * 缓存 key 拼装函数 + */ +func (k *Rule) getCacheKey(gets map[string]string) string { + // cacheKey，先加上 r101 形式的前缀，代表所属规则，101等同于规则returnCode + cacheKey := "r" + strconv.Itoa(int(k.returnCode)) + + kForSort := make([]string, len(k.keys)) + i := 0 + for keyName := range k.keys { + kForSort[i] = keyName + i++ + } + // 因map遍历时的顺序随机不定，但cache key必须确保一致； + // 故先对map的key字典序排序，后面按此顺序引用map值 + sort.Strings(kForSort) + + for _, keyName := range kForSort { + keyValue := k.keys[keyName] + switch keyValue.(type) { + case *GroupKey: + // combine的groupkey，不拼入，达到combine效果(联合计数) + if !keyValue.(*GroupKey).combine { + cacheKey = cacheKey + "|" + gets[keyName] + } + + default: + cacheKey = cacheKey + "|" + gets[keyName] + } + } + cacheKey = strings.Trim(cacheKey, "|") + return cacheKey +} + +/** + * 查询：查询规则当前的缓存值 + */ +func (k *Rule) getCacheValue(cacheKey string) (int, error) { + redisConn := RedisPool.Get() + defer redisConn.Close() + + var err error + var cacheValue int + cacheValue, err = redis.Int(redisConn.Do("GET", cacheKey)) + if err == redis.ErrNil { + // 此 key 不存在，直接返回 0 + return 0, nil + } + if err != nil { + return 0, err + } + //println(cacheKey, " --> value:", cacheValue) + return cacheValue, nil +} + +/** + * 浏览；count规则缓存查询、比较 + */ +func (k *Rule) countBrowse(cacheKey string) (bool, error) { + redisConn := RedisPool.Get() + defer redisConn.Close() + + var err error + var cacheValue int + cacheValue, err = redis.Int(redisConn.Do("GET", cacheKey)) + if err == redis.ErrNil { + // 此 key 不存在，直接返回通过 + return false, nil + } + if err != nil { + return false, err + } + //println(cacheKey, " --> value:", cacheValue) + if k.count == 0 || k.count > int32(cacheValue) { + return false, nil + } + return true, nil +} + +/** + * 更新；count规则缓存更新 + */ +func (k *Rule) countUpdate(cacheKey string) error { + redisConn := RedisPool.Get() + defer redisConn.Close() + + var exists int + var err error + if exists, err = redis.Int(redisConn.Do("EXISTS", cacheKey)); err != nil { + return err + } + // set new cache + if exists == 0 { + var expireTime int32 + // 86400 按照 自然天计算 过期时间 + if k.time == 86400 { + y, m, d := time.Now().Date() + loc, _ := time.LoadLocation("Asia/Shanghai") + dayEnd := time.Date(y, m, d, 23, 59, 59, 0, loc).Unix() + expireTime = int32(dayEnd - time.Now().Unix()) + } else { + expireTime = k.time + } + + if _, err := redis.String(redisConn.Do("SETEX", cacheKey, expireTime, 1)); err != nil { + return err + } + + return nil + } + + // update + if _, err := redis.Int(redisConn.Do("INCR", cacheKey)); err != nil { + return err + } + + return nil +} + +/** + * 浏览；base方法缓存查询、比较 + */ +func (k *Rule) baseBrowse(cacheKey string) (bool, error) { + redisConn := RedisPool.Get() + defer redisConn.Close() + + var err error + var cacheValue int + cacheValue, err = redis.Int(redisConn.Do("GET", cacheKey)) + if err == redis.ErrNil { + // 此 key 不存在，直接返回通过 + return false, nil + } + if err != nil { + return false, err + } + //println(cacheKey, " --> value:", cacheValue) + if k.base == 0 || k.base > int32(cacheValue) { + return false, nil + } + var cacheKeyTime = cacheKey + BaseKeySuffix + + cacheValue, err = redis.Int(redisConn.Do("GET", cacheKeyTime)) + if err == redis.ErrNil { + // 此 key 不存在，直接返回通过 + return false, nil + } + if err != nil { + return false, err + } + + //println(cacheKey_time, " --> value:", cacheValue) + if k.count == 0 || k.count > int32(cacheValue) { + return false, nil + } + return true, nil +} + +/** + * 更新；base方法缓存更新 + */ +func (k *Rule) baseUpdate(cacheKey string) error { + redisConn := RedisPool.Get() + defer redisConn.Close() + + exists, err := redis.Int(redisConn.Do("EXISTS", cacheKey)) + if err != nil { + return err + } + if exists == 0 { + y, m, d := time.Now().Date() + dayEnd := time.Date(y, m, d, 23, 59, 59, 0, time.UTC).Unix() + expireTime := int32(dayEnd - time.Now().Unix()) + if _, err = redis.String(redisConn.Do("SETEX", cacheKey, expireTime, 1)); err != nil { + return err + } + + return nil + } + + // update + var cacheValue int + if cacheValue, err = redis.Int(redisConn.Do("INCR", cacheKey)); err != nil { + return err + } + if k.base == 0 || k.base > int32(cacheValue) { + return nil + } + + var cacheKeyTime = cacheKey + BaseKeySuffix + if exists, err = redis.Int(redisConn.Do("EXISTS", cacheKeyTime)); err != nil { + return err + } + if exists == 0 { + if _, err = redis.String(redisConn.Do("SETEX", cacheKeyTime, k.time, 1)); err != nil { + return err + } + return nil + } + if _, err = redis.Int(redisConn.Do("INCR", cacheKeyTime)); err != nil { + return err + } + return nil +} + +/** + * leak模式--查询 + * + */ +func (k *Rule) leakBrowse(cacheKey string) (bool, error) { + redisConn := RedisPool.Get() + defer redisConn.Close() + + listLen, err := redis.Int(redisConn.Do("LLEN", cacheKey)) + if err != nil { + return false, err + } + if listLen == 0 || listLen <= int(k.count) { + return false, nil + } + + defer func() { + go k.leakClear(cacheKey, listLen) + }() + + now := time.Now().Unix() + var edgeElement int64 + if edgeElement, err = redis.Int64(redisConn.Do("LINDEX", cacheKey, k.count)); err != nil { + return false, err + } + if int32(now-edgeElement) <= k.time { + return true, nil + } + return false, nil +} + +/** + * leak模式--清理 + * 清理队尾过期多余元素 + */ +func (k *Rule) leakClear(cacheKey string, listLen int) { + redisConn := RedisPool.Get() + defer redisConn.Close() + + for listLen > int(k.count+1) { + if _, err := redis.Int64(redisConn.Do("RPOP", cacheKey)); err != nil { + return + } + listLen-- + } +} + +/** + * leak模式--更新 + */ +func (k *Rule) leakUpdate(cacheKey string) error { + redisConn := RedisPool.Get() + defer redisConn.Close() + + now := time.Now().Unix() + if _, err := redis.Int(redisConn.Do("LPUSH", cacheKey, now)); err != nil { + return err + } + if _, err := redis.Int(redisConn.Do("EXPIRE", cacheKey, 2*k.time)); err != nil { + return err + } + return nil +} + +/** + * leak模式--反馈 + * 根据指令，减少桶内若干元素 + */ +func (k *Rule) leakFeedback(cacheKey string, feedback int) error { + redisConn := RedisPool.Get() + defer redisConn.Close() + + for feedback > 0 { + if _, err := redis.Int64(redisConn.Do("LPOP", cacheKey)); err != nil { + return err + } + feedback-- + } + return nil +} + +/** + * 多重浏览；direct规则直接判定 + */ +func (k *Rule) multiDirectBrowse(cacheKeys []interface{}) (map[string]bool, error) { + return nil, nil +} + +/** + * 多重浏览；count规则缓存查询、比较 + */ +func (k *Rule) multiCountBrowse(cacheKeys []interface{}) (map[string]bool, error) { + redisConn := RedisPool.Get() + defer redisConn.Close() + + multiResult := make(map[string]bool, len(cacheKeys)) + cacheVals := make([]int, len(cacheKeys)) + intf := []interface{}{} + for i := range cacheVals { + intf = append(intf, &cacheVals[i]) + } + reply, err := redis.Values(redisConn.Do("MGET", cacheKeys...)) + if err != nil { + return nil, err + } + if _, err = redis.Scan(reply, intf...); err != nil { + return nil, err + } + + for i, v := range cacheVals { + key := cacheKeys[i].(string) + if k.count == 0 || v == 0 || k.count > int32(v) { + multiResult[key] = false + continue + } + multiResult[key] = true + } + return multiResult, nil +} + +/** + * 多重浏览；base方法缓存查询、比较 + */ +func (k *Rule) multiBaseBrowse(cacheKeys []interface{}) (map[string]bool, error) { + return nil, nil +} diff --git a/koala/policy.go b/koala/policy.go new file mode 100644 index 0000000..268cf6a --- /dev/null +++ b/koala/policy.go @@ -0,0 +1,260 @@ +/** + * Koala Rule Engine Core + * + * @package: main + * @desc: koala engine - Dict list parser + * + * @author: heiyeluren + * @github: https://github.com/heiyeluren + * @blog: https://blog.csdn.net/heiyeshuwu + * + */ + +package koala + +import ( + "encoding/json" + "errors" + "io/ioutil" + "strconv" + "strings" +) + +// RetValue retValue数据类型 +type RetValue struct { + RetType int32 + RetCode int32 + ErrNo int32 + ErrMsg string + StrReason string + NeedVcode int32 + VcodeLen int32 + VcodeType int32 + Other string + Version int32 +} + +// Policy . +// 策略结构，包含：dicts词表、rule规则、retValue 返回值三种数据; +// 其中 rule 是主体，dicts、和 retValue 会被 rule引用到 +type Policy struct { + dictsTable map[string]map[string]string + ruleTable []Rule + retValueTable map[int]RetValue +} + +/** + * 各种分隔符，trim的时候要除掉他们 + */ +const emptyRunes = " \r\t\v" + +var ( + // GlobalPolicy .全局策略配置 + GlobalPolicy *Policy + // TempPolicy .临时策略配置（用于策略的动态更新） + TempPolicy *Policy +) + +// NewPolicy Policy构造函数，完成各个元素的空间初始化 +func NewPolicy() *Policy { + return &Policy{ + dictsTable: make(map[string]map[string]string), + ruleTable: make([]Rule, 0, 50), + retValueTable: make(map[int]RetValue), + } +} + +// PolicyInterpreter Policy解释器，用于从文件解析配置，记录到 Policy 结构中 +func PolicyInterpreter(extStream string) error { + // temp策略缓冲区初始化 + TempPolicy = NewPolicy() + + var rawStream []byte + var err error + if extStream != "" { + rawStream = []byte(extStream) + } else { + rawStream, err = ioutil.ReadFile(Config.Get("rule_file")) + if err != nil { + return errors.New("cannot load rule file") + } + } + + DynamicUpdateFiles = append(DynamicUpdateFiles, Config.Get("rule_file")) + lines := strings.Split(string(rawStream), "\n") + + // 配置文件分成三部分 词表、规则和返回结果，起始字符串分别是 [dicts] [rules] [result] + // 寻找这三部分的起始行 + dictsPos, rulesPos, resultsPos := 0, 0, 0 + for index, line := range lines { + line = strings.Trim(line, emptyRunes) + if line == "" || line[0] == '#' { + continue + } + if strings.EqualFold(strings.Trim(line, emptyRunes), "[dicts]") { + dictsPos = index + } + if strings.EqualFold(strings.Trim(line, emptyRunes), "[rules]") { + rulesPos = index + } + if strings.EqualFold(strings.Trim(line, emptyRunes), "[result]") { + resultsPos = index + } + } + + // 解析词表配置 + for index := dictsPos + 1; index < rulesPos; index++ { + line := strings.Trim(lines[index], emptyRunes) + if line == "" || line[0] == '#' { + continue + } + if err = dictsBuilder(line); err != nil { + return errors.New(err.Error() + " ;AT-LINE-" + strconv.Itoa(index) + "; " + line) + } + } + + // 解析规则配置 + for index := rulesPos + 1; index < resultsPos; index++ { + line := strings.Trim(lines[index], emptyRunes) + if line == "" || line[0] == '#' { + continue + } + if err = rulesBuilder(line); err != nil { + return errors.New(err.Error() + " ;AT-LINE-" + strconv.Itoa(index) + "; " + line) + } + //println(line) + } + + // 解析返回结果配置 + for index := resultsPos + 1; index < len(lines); index++ { + line := strings.Trim(lines[index], emptyRunes) + if line == "" || line[0] == '#' { + continue + } + if err = resultsBuilder(line); err != nil { + return errors.New(err.Error() + " ;AT-LINE-" + strconv.Itoa(index) + "; " + line) + } + } + + // 校验规则有效性 + if err = ruleValidityCheck(); err != nil { + return err + } + + // 覆盖全局策略配置 + GlobalPolicy = TempPolicy + + return nil +} + +/** + * rule构造器，对单条 rule 进行解析 然后存入 TempPolicy + */ +func rulesBuilder(rule string) error { + // rule : [direct] [qid @ global_qid_whitelist] [time=1; count=0;] [result=1; return=101] + parts := strings.SplitN(rule, ":", 2) + if len(parts) == 2 && strings.EqualFold(strings.Trim(parts[0], emptyRunes), "rule") { + // + var singleRule Rule + if err := singleRule.Constructor(parts[1]); err != nil { + return err + } + TempPolicy.ruleTable = append(TempPolicy.ruleTable, singleRule) + } else { + return errors.New("rule syntax error: struct error") + } + return nil +} + +/** + * dicts构造器，对单条 dict 进行解析 然后存入 TempPolicy + */ +func dictsBuilder(dict string) error { + // 配置格式 名称 : 配置文件名 + // global_qid_whitelist : etc/global_qid_whitelist.dat + + parts := strings.SplitN(dict, ":", 2) + if len(parts) != 2 { + return errors.New("dict syntax error: struct error") + } + oneDict := make(map[string]string, 10) + + // 读取配置文件 + fileName := strings.Trim(parts[1], emptyRunes) + DynamicUpdateFiles = append(DynamicUpdateFiles, fileName) + rawStream, err := ioutil.ReadFile(fileName) + if err != nil { + return errors.New("cannot load dict file") + } + lines := strings.Split(string(rawStream), "\n") + for _, v := range lines { + item := strings.Trim(v, emptyRunes) + oneDict[item] = item + } + dictName := strings.Trim(parts[0], emptyRunes) + TempPolicy.dictsTable[dictName] = oneDict + return nil +} + +/** + * retValue 构造器，对单条 result 进行解析 + */ +func resultsBuilder(result string) error { + // 1 : { "Ret_type":1, "Ret_code" : 0, "Err_no":0, "Err_msg":"", "Str_reason":"Allow", "Need_vcode":0, "Vcode_len":0, "Vcode_type":0, "Other":"", "Version":0 } + parts := strings.SplitN(result, ":", 2) + var retType = -1 + var err error + if retType, err = strconv.Atoi(strings.Trim(parts[0], emptyRunes)); err != nil { + return err + } + + var ret RetValue + inString := strings.Trim(parts[1], emptyRunes) + if err = json.Unmarshal([]byte(inString), &ret); err != nil { + return err + } + //fmt.Printf("%+v \n", ret) + TempPolicy.retValueTable[retType] = ret + return nil +} + +// ruleValidityCheck . +// 功能：rule合法性检查（事后检查） +// 对象：TempPolicy +// 作用：此检查发生在，解析过程的末尾，对已经读入内存的配置，检查其合法性、逻辑正确性 +// a、return值 唯一性检查 +// b、base、count、time、result、return完整性检查 +// c、base、count、time、result、return的范围检查，如 0 值、负值等 +func ruleValidityCheck() error { + var returnMap = make(map[int32]string) + + for _, singleRule := range TempPolicy.ruleTable { + switch singleRule.method { + case "direct": + break + case "count": + if singleRule.count <= 0 || singleRule.time <= 0 { + return errors.New("rule semantic error: rule argument out of range") + } + case "base": + if singleRule.base <= 0 || singleRule.count <= 0 || singleRule.time <= 0 { + return errors.New("rule semantic error: rule argument out of range") + } + default: + } + + if singleRule.result <= 0 || singleRule.returnCode <= 0 { + return errors.New("rule semantic error: result invalid") + } + + if _, OK := TempPolicy.retValueTable[int(singleRule.result)]; !OK { + return errors.New("rule semantic error: result type no found") + } + + if _, OK := returnMap[singleRule.returnCode]; OK { + return errors.New("rule semantic error: rules with same return code") + } + returnMap[singleRule.returnCode] = "hi" + } + return nil +} diff --git a/koala/policyCounter.go b/koala/policyCounter.go new file mode 100644 index 0000000..9c1dd54 --- /dev/null +++ b/koala/policyCounter.go @@ -0,0 +1,124 @@ +/** + * Koala Rule Engine Core + * + * @package: main + * @desc: koala engine - Dict list parser & counter + * + * @author: heiyeluren + * @github: https://github.com/heiyeluren + * @blog: https://blog.csdn.net/heiyeshuwu + * + */ + +package koala + +import ( + "fmt" + "time" + + "github.com/heiyeluren/koala/utility" +) + +// CountMessage . +type CountMessage struct { + ruleNo int32 + decision int32 +} + +// Counter . +type Counter struct { + allow int64 + deny int64 +} + +const ( + // ALLOW .允许 + ALLOW = 1 + // DENY 拒绝 + DENY = 2 +) + +var ( + // PolicyCounter . + PolicyCounter map[string]map[int32]*Counter + // CountTransChannel . + CountTransChannel chan *CountMessage +) + +func init() { + PolicyCounter = make(map[string]map[int32]*Counter) + CountTransChannel = make(chan *CountMessage, 1024) +} + +// CounterAgent 从 channel 里获取策略结果，然后定期写入日志 channel +func CounterAgent() { + + for { + // 从channel获取一条countMsg（统计消息） + countMsg := <-CountTransChannel + + dateString := GetDateString(time.Now()) + + if _, OK := PolicyCounter[dateString]; !OK { + // 将过期的元素（上一时段的）转移存储，并从map摘除 + for k, v := range PolicyCounter { + recordExpiredCounter(v) + delete(PolicyCounter, k) + } + // 初始化当前时段map + PolicyCounter[dateString] = make(map[int32]*Counter) + } + if _, OK := PolicyCounter[dateString][countMsg.ruleNo]; !OK { + // 如果map元素不存在，则初始化此元素 + initCounter := new(Counter) + PolicyCounter[dateString][countMsg.ruleNo] = initCounter + } + + singleCounter := PolicyCounter[dateString][countMsg.ruleNo] + switch countMsg.decision { + case ALLOW: + singleCounter.allow += 1 + case DENY: + singleCounter.deny += 1 + default: + } + } +} + +/** + * 记录counter数据到日志 + */ +func recordExpiredCounter(counters map[int32]*Counter) { + logHandle := utility.NewLogger(time.Now().Format("20060102") + "_counter") + for k, v := range counters { + logMsg := fmt.Sprintf(" rule_no:%d allow:%d deny:%d ", k, v.allow, v.deny) + logHandle.Warning(logMsg) + } +} + +// CounterClient 统计API +func CounterClient(ruleNo int32, deny bool) { + msg := new(CountMessage) + msg.ruleNo = ruleNo + if deny { + msg.decision = DENY + } else { + msg.decision = ALLOW + } + + // 将统计消息，发给channal + CountTransChannel <- msg +} + +// GetDateString . +func GetDateString(t time.Time) string { + // 按天划分时段 + tString := t.Format("20060102") + return tString +} + +// GetCurrentCounters . +func GetCurrentCounters() map[int32]*Counter { + dateString := GetDateString(time.Now()) + return PolicyCounter[dateString] +} diff --git a/koala/policyLoader.go b/koala/policyLoader.go new file mode 100644 index 0000000..426ea7c --- /dev/null +++ b/koala/policyLoader.go @@ -0,0 +1,87 @@ +/** + * Koala Rule Engine Core + * + * @package: main + * @desc: koala engine - Dict list parser & loader + * + * @author: heiyeluren + * @github: https://github.com/heiyeluren + * @blog: https://blog.csdn.net/heiyeshuwu + * + */ + +package koala + +import ( + "bytes" + "crypto/md5" + "errors" + "fmt" + "io" + "io/ioutil" + "time" + + "github.com/heiyeluren/koala/utility" +) + +// NewPolicyMD5 PolicyMd5 初始化函数 +func NewPolicyMD5() string { + m, err := PolicyMD5Str() + if err != nil { + // error log + return "" + } + return m +} + +// PolicyLoader . +// policy实时更新函数 +// 说明：用于实时更新rule配置，解析过程调用PolicyInterpreter()处理 +func PolicyLoader() { + var err error + var m string + logHandle := utility.NewLogger("") + + for { + d := Config.GetInt("policy_loader_frequency") + if d == 0 { + d = 300 + } + time.Sleep(time.Duration(d) * time.Second) + + m, err = PolicyMD5Str() + if err != nil { + logHandle.Warning("[errmsg=" + err.Error() + " md5=" + PolicyMd5 + "]") + } + //println(Policy_md5) + + if m != PolicyMd5 { + DynamicUpdateFiles = []string{} + err = PolicyInterpreter("") + if err != nil { + logHandle.Warning("[errmsg=" + err.Error() + "]") + } else { + PolicyMd5 = m + logHandle.Trace("[msg=policy reload! new-md5=" + PolicyMd5 + "]") + } + } + } +} + +// PolicyMD5Str 计算 DynamicUpdateFiles 所包含文件的 md5 值 +func PolicyMD5Str() (string, error) { + var contentStream bytes.Buffer + for _, file := range DynamicUpdateFiles { + rawStream, err := ioutil.ReadFile(file) + if err != nil { + return "", errors.New("cannot load policy file") + } + contentStream.Write(rawStream) + } + + filesMd5 := md5.New() + io.WriteString(filesMd5, contentStream.String()) + + stringMd5 := fmt.Sprintf("%x", filesMd5.Sum(nil)) + return stringMd5, nil +} diff --git a/koala/service.go b/koala/service.go new file mode 100644 index 0000000..5ce35d7 --- /dev/null +++ b/koala/service.go @@ -0,0 +1,478 @@ +/** + * Koala Rule Engine Core + * + * @package: main + * @desc: koala engine - Http api access process + * + * @author: heiyeluren + * @github: https://github.com/heiyeluren + * @blog: https://blog.csdn.net/heiyeshuwu + * + */ + +package koala + +import ( + "encoding/json" + "net/url" + "strconv" + "strings" + + "github.com/heiyeluren/koala/utility" +) + +// DoRuleBrowse 查询访问接口 +func (s *FrontServer) DoRuleBrowse(request *utility.HttpRequest, response *utility.HttpResponse, logHandle *utility.Logger) { + // 本地策略指针，可避免匹配过程中Global策略被替换 导致不一致 + var localPolicy = GlobalPolicy + + var singleRule Rule + var err error + var retValue = localPolicy.retValueTable[0] + // 匹配每一条rule规则 + for _, singleRule = range localPolicy.ruleTable { + var satisfied = true + // 遍历每个key，若不匹配或者参数未传，不命中 + for k, v := range singleRule.keys { + str := request.Gstr(k) + if str != "" && v.matches(str) { + continue + } + satisfied = false + break + } + if !satisfied { + continue + } + + // 对命中的key，查缓存值，与阀值比较，判断是否超出限制 + var isOut bool + ruleCacheKey := singleRule.getCacheKey(request.Gets()) + //println(ruleCacheKey) + switch singleRule.method { + case "direct": + isOut = true + case "count": + if isOut, err = singleRule.countBrowse(ruleCacheKey); err != nil { + logHandle.Fatal("[errmsg=" + err.Error() + "]") + } + case "base": + if isOut, err = singleRule.baseBrowse(ruleCacheKey); err != nil { + logHandle.Fatal("[errmsg=" + err.Error() + "]") + } + case "leak": + if isOut, err = singleRule.leakBrowse(ruleCacheKey); err != nil { + logHandle.Fatal("[errmsg=" + err.Error() + "]") + } + default: + } + + // 统计，记录策略判定数据 + CounterClient(singleRule.returnCode, isOut) + + // 超出限制，按照rule的约定，给出处置策略 + if isOut { + retValue = localPolicy.retValueTable[int(singleRule.result)] + retValue.RetCode = singleRule.returnCode + break + } + retValue = localPolicy.retValueTable[1] + } + + // _writeThrough“直接写缓存”开关，同时完成 Browse和 Update两步操作。 + if retValue.RetType >= 1 && request.Gstr("_writeThrough") == "yes" { + go RuleUpdateLogic(request, logHandle) + } + + // 返回json结果 + var retString []byte + retString, err = json.Marshal(retValue) + if err != nil { + response.SetCode(500) + return + } + response.Puts(string(retString)) + + response.SetCode(200) +} + +// DoRuleUpdate 更新访问接口 +func (s *FrontServer) DoRuleUpdate(request *utility.HttpRequest, response *utility.HttpResponse, logHandle *utility.Logger) { + go RuleUpdateLogic(request, logHandle) + + response.Puts(`{"err_no":0, "err_msg":"OK"}`) + response.SetCode(200) +} + +/** + * 查询缓存状态接口 + */ +/* +func (s *FrontServer) DoRuleValue(request *network.HttpRequest, response *network.HttpResponse, logHandle *logger.Logger) { + var localPolicy *Policy = GlobalPolicy + + var singleRule KoalaRule + var err error + var cacheVal int = -1 + var req_rule int32 = int32(request.Gint("rule_no")) + for _, singleRule = range localPolicy.ruleTable { + if singleRule.returnCode != req_rule { + continue + } + ruleCacheKey := singleRule.getCacheKey(request.Gets()) + if cacheVal, err = singleRule.getCacheValue(ruleCacheKey); err != nil { + logHandle.Fatal("[errmsg=" + err.Error() + "]") + response.Puts(`{"err_no":-1, "err_msg":"system error!"}`) + response.SetCode(500) + break + } + } + if cacheVal == -1 { + response.Puts(`{"err_no":-2, "err_msg":"unknown rule_no!"}`) + response.SetCode(400) + } else { + response.Puts(`{"err_no":0, "err_msg":"", "cache_val":` + strconv.Itoa(cacheVal) + `}`) + response.SetCode(200) + } +} +*/ + +// DoRuleBrowseComplete 非中断查询接口（可命中、并返回多条策略） +func (s *FrontServer) DoRuleBrowseComplete(request *utility.HttpRequest, response *utility.HttpResponse, logHandle *utility.Logger) { + // 本地策略指针，可避免匹配过程中Global策略被替换 导致不一致 + var localPolicy *Policy = GlobalPolicy + + var singleRule Rule + var err error + // 用于返回多个结果，RetValue数组 + var retArray []RetValue + var retValue = localPolicy.retValueTable[0] + // 匹配每一条rule规则 + for _, singleRule = range localPolicy.ruleTable { + var satisfied = true + // 遍历每个key，若不匹配或者参数未传，跳过 + for k, v := range singleRule.keys { + str := request.Gstr(k) + if str != "" && v.matches(str) { + continue + } + satisfied = false + break + } + if !satisfied { + continue + } + + // 对匹配的key，查缓存值，与阀值比较，判断是否超出限制 + var isOut bool + switch singleRule.method { + case "direct": + isOut = true + case "count": + ruleCacheKey := singleRule.getCacheKey(request.Gets()) + if isOut, err = singleRule.countBrowse(ruleCacheKey); err != nil { + logHandle.Fatal("[errmsg=" + err.Error() + "]") + } + case "base": + ruleCacheKey := singleRule.getCacheKey(request.Gets()) + if isOut, err = singleRule.baseBrowse(ruleCacheKey); err != nil { + logHandle.Fatal("[errmsg=" + err.Error() + "]") + } + default: + } + + // 统计，记录策略判定数据 + CounterClient(singleRule.returnCode, isOut) + + // 命中，拼装结果 + if isOut { + retValue = localPolicy.retValueTable[int(singleRule.result)] + retValue.RetCode = singleRule.returnCode + retArray = append(retArray, retValue) + } + } + + // 如果没有命中任何策略，返回默认值 + if retValue.RetType == 0 { + retArray = append(retArray, retValue) + } + + // _writeThrough“直接写缓存”开关，同时完成 Browse和 Update两步操作。 + if retValue.RetType <= 1 && request.Gstr("_writeThrough") == "yes" { + RuleUpdateLogic(request, logHandle) + } + + // 返回json结果 + var retString []byte + retString, err = json.Marshal(retArray) + if err != nil { + response.SetCode(500) + return + } + response.Puts(string(retString)) + + response.SetCode(200) +} + +/** + * 反馈-feedback-接口 + */ +/* +func (s *FrontServer) DoRuleFeedback(request *network.HttpRequest, response *network.HttpResponse, logHandle *logger.Logger) { + + var localPolicy *Policy = GlobalPolicy + + var singleRule KoalaRule + var feedbackType = request.Gstr("_feedbackType") + // 匹配每一条rule规则 + for _, singleRule = range localPolicy.ruleTable { + var satisfied = true + // 遍历每个key，若不匹配或者参数未传，不命中 + for k, v := range singleRule.keys { + str := request.Gstr(k) + if str != "" && v.matches(str) { + continue + } + satisfied = false + break + } + if !satisfied { + continue + } + + var feedback int32 + switch feedbackType { + case "erase1": + feedback = singleRule.erase1 + case "erase2": + feedback = singleRule.erase2 + default: + feedback = singleRule.erase1 + } + + ruleCacheKey := singleRule.getCacheKey(request.Gets()) + go singleRule.leakFeedback(ruleCacheKey, int(feedback)) + } + + response.Puts(`{"err_no":0, "err_msg":"OK"}`) + response.SetCode(200) +} +*/ + +// RuleUpdateLogic 更新操作执行函数 +func RuleUpdateLogic(request *utility.HttpRequest, logHandle *utility.Logger) { + // 本地策略指针，可避免匹配过程中Global策略被替换 导致不一致 + var localPolicy *Policy = GlobalPolicy + + // 匹配每一条rule规则 + var singleRule Rule + for _, singleRule = range localPolicy.ruleTable { + var satisfied = true + // 遍历每个key，若不匹配或者参数未传，不命中 + for k, v := range singleRule.keys { + s := request.Gstr(k) + if s != "" && v.matches(s) { + continue + } + satisfied = false + break + } + if !satisfied { + continue + } + + ruleCacheKey := singleRule.getCacheKey(request.Gets()) + // 更新cache值 + switch singleRule.method { + case "count": + if singleRule.count == 0 { + continue + } + if err := singleRule.countUpdate(ruleCacheKey); err != nil { + logHandle.Fatal("[errmsg=" + err.Error() + "]") + continue + } + case "base": + if err := singleRule.baseUpdate(ruleCacheKey); err != nil { + logHandle.Fatal("[errmsg=" + err.Error() + "]") + continue + } + case "leak": + if err := singleRule.leakUpdate(ruleCacheKey); err != nil { + logHandle.Fatal("[errmsg=" + err.Error() + "]") + continue + } + default: + } + } +} + +// DoMonitorAlive 监控连接redis是否成功 +func (s *FrontServer) DoMonitorAlive(request *utility.HttpRequest, response *utility.HttpResponse, logHandle *utility.Logger) { + redisConn := RedisPool.Get() + defer redisConn.Close() + if _, err := redisConn.Do("PING"); err != nil { + response.Puts(`{"errno": -1, "errmsg": "redis-error!"}`) + response.SetCode(500) + logHandle.Fatal("[errmsg=" + err.Error() + "]") + return + } + response.Puts(`{"errno": 0, "errmsg": "OK!"}`) + response.SetCode(200) +} + +// Job . +type Job struct { + ID string + Arg string +} + +// JobResult . +type JobResult struct { + ID string + Result RetValue +} + +// JobBuffer . +type JobBuffer struct { + ID string + args map[string]string + key string + status bool + decision int + retCode int32 +} + +// DoMultiBrowse 多重浏览访问接口 +func (s *FrontServer) DoMultiBrowse(request *utility.HttpRequest, response *utility.HttpResponse, logHandle *utility.Logger) { + argsJSON := request.Gstr("argsJson") + if argsJSON == "" { + response.SetCode(400) + return + } + + var jobs []Job + err := json.Unmarshal([]byte(argsJSON), &jobs) + if err != nil { + logHandle.Fatal("[errmsg=" + err.Error() + "]") + response.SetCode(400) + return + } + + var logMsg string = "" + logMsg += "[ cip=" + request.GetRemoteIP() + logMsg += " intf=" + request.PathInfo() + + var buffers []JobBuffer + //var job Job + for _, job := range jobs { + var buf JobBuffer + buf.ID = job.ID + buf.args = parseJobArgs(job.Arg) + buf.status = false + buf.decision = 0 + buffers = append(buffers, buf) + logMsg += " ID" + job.ID + "@" + job.Arg + } + logMsg += " ] [" + + var localPolicy = GlobalPolicy + var singleRule Rule + for _, singleRule = range localPolicy.ruleTable { + var cacheKeys []interface{} + for i, buf := range buffers { + buffers[i].key = "" + var satisfied = true + for k, v := range singleRule.keys { + str, OK := buf.args[k] + if OK && v.matches(str) { + continue + } + satisfied = false + break + } + if satisfied && !buf.status { + buffers[i].key = singleRule.getCacheKey(buf.args) + cacheKeys = append(cacheKeys, buffers[i].key) + } + } + + if len(cacheKeys) == 0 { + continue + } + + var multiResult map[string]bool + switch singleRule.method { + case "direct": + if multiResult, err = singleRule.multiDirectBrowse(cacheKeys); err != nil { + // err log + } + case "count": + if multiResult, err = singleRule.multiCountBrowse(cacheKeys); err != nil { + logHandle.Fatal("[errmsg=" + err.Error() + "]") + } + case "base": + if multiResult, err = singleRule.multiBaseBrowse(cacheKeys); err != nil { + // err log + } + default: + } + + // 统计，记录策略判定数据 + for _, decision := range multiResult { + CounterClient(singleRule.returnCode, decision) + } + + for i, buf := range buffers { + isOut, OK := multiResult[buf.key] + if OK && isOut { + buffers[i].status = true + buffers[i].decision = int(singleRule.result) + buffers[i].retCode = singleRule.returnCode + } else if OK && !buf.status { + buffers[i].decision = 1 + } + } + } + + var jobResults []JobResult + for _, buf := range buffers { + var singleResult JobResult + singleResult.ID = buf.ID + singleResult.Result = localPolicy.retValueTable[buf.decision] + singleResult.Result.RetCode = buf.retCode + jobResults = append(jobResults, singleResult) + logMsg += " ID" + buf.ID + "~Ret_code:" + strconv.Itoa(int(buf.retCode)) + } + logMsg += " ]" + logHandle.Notice(logMsg) + + // 返回json结果 + var retString []byte + retString, err = json.Marshal(jobResults) + if err != nil { + response.SetCode(500) + return + } + response.Puts(string(retString)) + + response.SetCode(200) + +} + +func parseJobArgs(rawArg string) map[string]string { + retMap := make(map[string]string, 0) + argString, err := url.QueryUnescape(rawArg) + if err != nil { + return nil + } + kvStrings := strings.Split(argString, "&") + for _, kvString := range kvStrings { + parts := strings.SplitN(kvString, "=", 2) + if len(parts) != 2 { + return nil + } + retMap[parts[0]] = parts[1] + } + return retMap +} diff --git a/koala/util.go b/koala/util.go new file mode 100644 index 0000000..6a29fb9 --- /dev/null +++ b/koala/util.go @@ -0,0 +1,61 @@ +/** + * Koala Rule Engine Core + * + * @package: main + * @desc: koala engine - Utils functions + * + * @author: heiyeluren + * @github: https://github.com/heiyeluren + * @blog: https://blog.csdn.net/heiyeshuwu + * + */ + +package koala + +import ( + "errors" + "io/ioutil" + "os" + "strconv" + "strings" +) + +// SavePid 进程 pid 记录函数 +func SavePid(pidFile string) { + pid := os.Getpid() + pidString := strconv.Itoa(pid) + ioutil.WriteFile(pidFile, []byte(pidString), 0777) +} + +// IsIPAddress 判断字符串是否是 IP 地址 +func IsIPAddress(s string) bool { + // 符合x.x.x.x模式的字符串，即认为是ip地址 + parts := strings.Split(s, ".") + return len(parts) == 4 +} + +// ToInteger64 将代表ip地址、纯数字的string值 都统一转换成int64值 +func ToInteger64(s string) (int64, error) { + var err error + var result int64 = 0 + if IsIPAddress(s) { + parts := strings.Split(s, ".") + for i, v := range parts { + var intVal int + if intVal, err = strconv.Atoi(v); err != nil { + return result, err + } + if intVal > 255 || intVal < 0 { + return result, errors.New("rule syntax error: scope error,out of ip range 0-255") + } + result += int64(intVal) << uint((3-i)*8) + } + } else { + var int64Val int64 + if int64Val, err = strconv.ParseInt(s, 10, 64); err != nil { + return 0, err + } + result = int64Val + } + return result, nil +} diff --git a/lib/README b/lib/README deleted file mode 100644 index fb4ba52..0000000 --- a/lib/README +++ /dev/null @@ -1,7 +0,0 @@ -Redis客户端库依赖于： -github.com/garyburd/redigo/redis - - -说明： -实际使用中，可以使用新版本的 github.com/go-redis/redis 库进行取代，需要自己调整部分import库。 - diff --git a/lib/garyburd.tar.gz b/lib/garyburd.tar.gz deleted file mode 100644 index c04663c..0000000 Binary files a/lib/garyburd.tar.gz and /dev/null differ diff --git a/output/koala_online.tar.gz b/output/koala_online.tar.gz index d4302db..c1df122 100644 Binary files a/output/koala_online.tar.gz and b/output/koala_online.tar.gz differ diff --git a/sdk/koala_sdk.php b/sdk/koala_sdk.php index 86e4727..2bd18bd 100644 --- a/sdk/koala_sdk.php +++ b/sdk/koala_sdk.php @@ -161,7 +161,7 @@ public function write($param) /* monitorAlive * * 用途： koala服务的监控接口，能保证 ： koala存活，以及koala与redis直接的 “连通性” - * 接口url： GET /rule/update + * 接口url： GET /monitor/alive * * @ 返回值： * errno 错误码 diff --git a/src/koala/KoalaKey.go b/src/koala/KoalaKey.go deleted file mode 100644 index bcab00c..0000000 --- a/src/koala/KoalaKey.go +++ /dev/null @@ -1,276 +0,0 @@ -/** - * Koala Rule Engine Core - * - * @package: main - * @desc: koala engine code - * - * @author: heiyeluren - * @github: https://github.com/heiyeluren - * @blog: https://blog.csdn.net/heiyeshuwu - * - */ - -package main - -import ( - "errors" - "strconv" - "strings" -) - -/** - * KoalaKey interface ；用于支持多种 key 类型 - */ -type KoalaKey interface { - // 从配置字符串识别并生成 key 结构 - build(op, k, v string) error - - // 匹配 s 是否包含于 key 的范围 - matches(s string) bool - - // 仅调试用途 - dump() string -} - -// KoalaKey 有两个子类型 -// 集合 GroupKey -// 范围 RangeKey - -/** - * 集合 key 类型；满足 KoalaKey interface - */ -type GroupKey struct { - set map[string]string - inverse bool // 取反标记；@,!@ - combine bool // 合并标记；{~} -} - -/** - * dump() - */ -func (this *GroupKey) dump() string { - ret := "" - if this.inverse { - ret += "!" - } - if this.combine { - ret += "~" - } - for v, _ := range this.set { - ret += v + "," - } - return ret -} - -/** - * build() - */ -func (this *GroupKey) build(sp, k, v string) error { - // 词表识别 - this.set = make(map[string]string, 10) - this.combine = false - this.inverse = strings.HasSuffix(k, "!") - var isPresent bool - v = strings.Trim(v, emptyRunes) - if sp == "@" { - this.set, isPresent = TempPolicy.dictsTable[v] - if !isPresent { - return errors.New("rule build error: Dict not present!") - } - return nil - } - if sp == "=" { - this.combine = strings.HasSuffix(v, "{~}") - v = strings.Trim(v, "{~}") - elements := strings.Split(v, ",") - for _, e := range elements { - item := strings.Trim(e, emptyRunes) - this.set[item] = item - } - } - return nil -} - -/** - * matches() - */ -func (this *GroupKey) matches(s string) bool { - s = strings.Trim(s, emptyRunes) - if _, OK := this.set[s]; OK != this.inverse { - return true - } - return false -} - -/** - * 范围key；一个范围key可以包含多个范围区间 - */ -type RangeKey struct { - scopes []*Scope - inverse bool // 取反标记 ! -} - -/** - * dump - */ -func (this *RangeKey) dump() string { - ret := "" - if this.inverse { - ret += "!" - } - for _, scop := range this.scopes { - ret += scop.dump() + "&" - } - return ret -} - -/** - * build - */ -func (this *RangeKey) build(sp, k, v string) error { - // 范围识别 - var err error = nil - var int64Val int64 - this.inverse = strings.HasSuffix(k, "!") - v = strings.Trim(v, emptyRunes) - if sp == "<" { - oneScope := new(Scope) - oneScope.op = sp - if int64Val, err = strconv.ParseInt(v, 10, 64); err != nil { - return errors.New("rule syntax error: < error,not integer!") - } - oneScope.start = int64Val - this.scopes = []*Scope{oneScope} - return nil - } - if sp == ">" { - oneScope := new(Scope) - oneScope.op = sp - if int64Val, err = strconv.ParseInt(v, 10, 64); err != nil { - return errors.New("rule syntax error: > error,not integer!") - } - oneScope.end = int64Val - this.scopes = []*Scope{oneScope} - return nil - } - // sp 是 = - if v == "+" { - oneScope := new(Scope) - oneScope.op = v - this.scopes = []*Scope{oneScope} - return nil - } - this.scopes = []*Scope{} - parts := strings.Split(v, ",") - for _, sc := range parts { - oneScope := new(Scope) - if err = oneScope.build(sc); err != nil { - return err - } - this.scopes = append(this.scopes, oneScope) - } - return nil -} - -/** - * matches - */ -func (this *RangeKey) matches(s string) bool { - // + 号，任意值逻辑，直接matche - for _, sco := range this.scopes { - if sco.op == "+" { - return true - } - } - int64val, err := ToInteger64(s) - if err != nil { - return false - } - isIn := false - for _, sco := range this.scopes { - if sco.matches(int64val) { - isIn = true - break - } - } - - return isIn != this.inverse -} - -/** - * 范围 scope 类型，标识一个数值区间，如：>100, 1-9 - */ -type Scope struct { - op string // -,+,>,< - start int64 - end int64 -} - -/** - * dump - */ -func (this *Scope) dump() string { - a := strconv.FormatInt(this.start, 10) - b := strconv.FormatInt(this.end, 10) - return this.op + "^" + a + "^" + b -} - -/** - * build - */ -func (this *Scope) build(sc string) error { - var err error = nil - this.op = "-" - parts := strings.Split(sc, "-") - if len(parts) == 2 { - this.start, err = ToInteger64(strings.Trim(parts[0], emptyRunes)) - if err != nil { - return err - } - this.end, err = ToInteger64(strings.Trim(parts[1], emptyRunes)) - if err != nil { - return err - } - return nil - } - if strings.ContainsAny(sc, "*") && IsIPAddress(sc) { - rep := strings.NewReplacer("*", "0") - add_start := strings.Trim(rep.Replace(sc), emptyRunes) - this.start, err = ToInteger64(add_start) - if err != nil { - return err - } - rep = strings.NewReplacer("*", "255") - add_end := strings.Trim(rep.Replace(sc), emptyRunes) - this.end, err = ToInteger64(add_end) - if err != nil { - return err - } - return nil - } - return errors.New("rule syntax error: scope error!") -} - -/** - * matches - */ -func (this *Scope) matches(v int64) bool { - switch this.op { - case "+": - return true - case "<": - if v < this.start { - return true - } - case ">": - if v > this.end { - return true - } - case "-": - if v >= this.start && v <= this.end { - return true - } - default: - } - return false -} diff --git a/src/koala/cacheSvc.go b/src/koala/cacheSvc.go deleted file mode 100644 index 1f1ddd8..0000000 --- a/src/koala/cacheSvc.go +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Koala Rule Engine Core - * - * @package: main - * @desc: koala engine - Redis cache conn pool - * - * @author: heiyeluren - * @github: https://github.com/heiyeluren - * @blog: https://blog.csdn.net/heiyeshuwu - * - */ - -package main - -import ( - "github.com/garyburd/redigo/redis" - "time" -) - -/** - * redis连接池初始化函数 - */ -func initredisPool() { - // redis服务 host:port - var server string = Config.Get("redis_server") - // redis服务 口令 - var password string = Config.Get("redis_auth") - // redis连接池 最大空闲连接数 - var maxIdle int = Config.GetInt("redis_pool_maxIdle") - // redis连接池 空闲连接超时时长 - var idleTimeout int = Config.GetInt("redis_pool_idleTimeout") - - var connectTimeout time.Duration = time.Duration(Config.GetInt("externalConnTimeout")) * time.Millisecond - var readTimeout time.Duration = time.Duration(Config.GetInt("externalReadTimeout")) * time.Millisecond - var writeTimeout time.Duration = time.Duration(Config.GetInt("externalWriteTimeout")) * time.Millisecond - - // 新建连接池 - RedisPool = &redis.Pool{ - MaxIdle: maxIdle, - IdleTimeout: time.Duration(idleTimeout) * time.Second, - // dial方法 - Dial: func() (redis.Conn, error) { - c, err := redis.DialTimeout("tcp", server, connectTimeout, readTimeout, writeTimeout) - if err != nil { - return nil, err - } - if _, err := c.Do("AUTH", password); err != nil { - c.Close() - return nil, err - } - return c, err - }, - // 连接可用性检查 方法 - TestOnBorrow: func(c redis.Conn, t time.Time) error { - _, err := c.Do("PING") - return err - }, - } -} diff --git a/src/koala/config.go b/src/koala/config.go deleted file mode 100644 index f045b9d..0000000 --- a/src/koala/config.go +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Koala Rule Engine Core - * - * @package: main - * @desc: koala engine - koala server config parse - * - * @author: heiyeluren - * @github: https://github.com/heiyeluren - * @blog: https://blog.csdn.net/heiyeshuwu - * - */ - -package main - -import ( - "flag" - "utility/configs" -) - -// 启动前加载配置文件 -func newConfig() *configs.Config { - var F string - flag.StringVar(&F, "f", "", "config file") - flag.Parse() - if F == "" { - panic("usage: ./koala -f etc/koala.conf") - } - Config := configs.NewConfig() - if err := Config.Load(F); err != nil { - panic(err.Error()) - } - return Config -} diff --git a/src/koala/httpFront.go b/src/koala/httpFront.go deleted file mode 100644 index 3dece88..0000000 --- a/src/koala/httpFront.go +++ /dev/null @@ -1,105 +0,0 @@ -/** - * Koala Rule Engine Core - * - * @package: main - * @desc: koala engine - Http server front api - * - * @author: heiyeluren - * @github: https://github.com/heiyeluren - * @blog: https://blog.csdn.net/heiyeshuwu - * - */ - -package main - -import ( - "reflect" - "regexp" - "strings" - "time" - "utility/logger" - "utility/network" -) - -var frontPattern = regexp.MustCompile("^[a-z][0-9a-z_]*/[a-z][0-9a-z_]*$") - -type FrontServer struct{} - -func NewFrontServer() *FrontServer { - return &FrontServer{} -} - -func FrontListen() { - tcpListener, err := network.TcpListen(Config.Get("listen")) - if err != nil { - panic(err.Error()) - } - for { - tcpConnection, err := tcpListener.Accept() - if err != nil { - continue - } - go FrontDispatch(network.NewHttpConnection(tcpConnection)) - } -} - -func FrontDispatch(httpConnection *network.HttpConnection) { - defer httpConnection.Close() - - request, err := httpConnection.ReadRequest(time.Duration(Config.GetInt("externalReadTimeout")) * time.Millisecond) - if err != nil { - return - } - response := network.NewHttpResponse() - - // 生成log句柄 - logHandle := logger.NewLogger("") - - pathInfo := strings.Trim(request.PathInfo(), "/") - parts := strings.Split(pathInfo, "/") - if len(parts) == 2 && frontPattern.Match([]byte(pathInfo)) { - methodName := "Do" + strings.Title(parts[0]) + strings.Title(parts[1]) - frontServer := NewFrontServer() - frontServerValue := reflect.ValueOf(frontServer) - methodValue := frontServerValue.MethodByName(methodName) - if methodValue.IsValid() { - requestValue := reflect.ValueOf(request) - responseValue := reflect.ValueOf(response) - logHandleValue := reflect.ValueOf(logHandle) - methodValue.Call([]reflect.Value{requestValue, responseValue, logHandleValue}) - goto finished - } - } - response.SetCode(404) - response.Puts("not found") -finished: - // 每个请求，记录访问日志 - requestLogWrite(request, response, logHandle) - - response.SetHeader("Connection", "close") - httpConnection.WriteResponse(response, time.Duration(Config.GetInt("externalWriteTimeout"))*time.Millisecond) -} - -func requestLogWrite(request *network.HttpRequest, response *network.HttpResponse, logHandle *logger.Logger) { - if request.PathInfo() == "/multi/browse" { - // 批量接口，不在此记录notice，在接口内部记录 - return - } - - var logMsg string = "" - logMsg += "[ cip=" + request.GetRemoteIP() - logMsg += " intf=" + request.PathInfo() - - gets := request.Gets() - for k, v := range gets { - logMsg += " " + k + "=" + v - } - posts := request.Posts() - for k, v := range posts { - logMsg += " " + k + "=" + v - } - logMsg += " ]" - logMsg += " [ BodyString=" + response.BodyString() + " ]" - - logHandle.Notice(logMsg) -} diff --git a/src/koala/koalaRule.go b/src/koala/koalaRule.go deleted file mode 100644 index 7b12a16..0000000 --- a/src/koala/koalaRule.go +++ /dev/null @@ -1,542 +0,0 @@ -/** - * Koala Rule Engine Core - * - * @package: main - * @desc: koala engine - Core Rule Parse engine - * - * @author: heiyeluren - * @github: https://github.com/heiyeluren - * @blog: https://blog.csdn.net/heiyeshuwu - * - */ - -package main - -import ( - "errors" - "github.com/garyburd/redigo/redis" - "sort" - "strconv" - "strings" - "time" -) - -const ( - // base附加cache key的后缀；在getCacheKey()的key后追加 - BASE_KEY_SUFFIX = "_B" -) - -/** - * rule类型 - */ -type KoalaRule struct { - methord string //只能为如下四个字符串 count base direct leak - keys map[string]KoalaKey - base int32 - time int32 - count int32 - erase1 int32 - erase2 int32 - result int32 - returnCode int32 -} - -/************************************************************ - KoalaRule 构建，build，相关方法 -************************************************************/ - -/** - * KoalaRule 的构造器 - */ -func (this *KoalaRule) Constructor(r string) error { - // [direct] [qid @ global_qid_whitelist] [time=1; count=0;] [result=1; return=101] - // [count] [act=ask;qid=+;] [time=2; count=1;] [result=2; return=201] - // [base] [act=ask;ip=+;] [base=50; time=10; count=1;] [result=2; return=203] - sections := strings.Split(r, "] [") - if len(sections) != 4 { - return errors.New("rule syntax error: section error!") - } - for i, _ := range sections { - sections[i] = strings.Trim(sections[i], emptyRunes+"[]") - } - this.methord = sections[0] - if this.methord != "count" && this.methord != "base" && this.methord != "direct" && this.methord != "leak" { - return errors.New("rule syntax error: methord error!") - } - this.keys = make(map[string]KoalaKey, 10) - if err := this.getKeys(sections[1]); err != nil { - return err - } - // 解析剩余的 count、returnCode 等 参数 - if err := this.getCountAndRet(sections[2], sections[3]); err != nil { - return err - } - return nil -} - -/** - * 解析 KoalaRule 的 count、returnCode 等 参数 - */ -func (this *KoalaRule) getCountAndRet(val, ret string) error { - val = strings.Trim(val, emptyRunes+";") - ret = strings.Trim(ret, emptyRunes+";") - vals := strings.Split(val, ";") - for _, s := range vals { - s = strings.Trim(s, emptyRunes) - parts := strings.SplitN(s, "=", 2) - if len(parts) != 2 { - return errors.New("rule syntax error: value error!") - } - valueName := strings.Trim(parts[0], emptyRunes) - valueData, err := strconv.Atoi(strings.Trim(parts[1], emptyRunes)) - if err != nil { - return errors.New("rule syntax error: value error!") - } - switch valueName { - case "base": - this.base = int32(valueData) - case "time": - this.time = int32(valueData) - case "count": - this.count = int32(valueData) - case "erase1": - this.erase1 = int32(valueData) - case "erase2": - this.erase2 = int32(valueData) - default: - return errors.New("rule syntax error: value error!") - } - } - rets := strings.Split(ret, ";") - for _, s := range rets { - s = strings.Trim(s, emptyRunes) - parts := strings.SplitN(s, "=", 2) - if len(parts) != 2 { - return errors.New("rule syntax error: value error!") - } - valueName := strings.Trim(parts[0], emptyRunes) - valueData, err := strconv.Atoi(strings.Trim(parts[1], emptyRunes)) - if err != nil { - return errors.New("rule syntax error: value error!") - } - switch valueName { - case "result": - this.result = int32(valueData) - case "return": - this.returnCode = int32(valueData) - default: - return errors.New("rule syntax error: value error!") - } - } - return nil -} - -/** - * 解析 KoalaRule 的 keys 参数 - */ -func (this *KoalaRule) getKeys(k string) error { - // act=ask;ip=+; - k = strings.Trim(k, emptyRunes+";") - allKey := strings.Split(k, ";") - for i, _ := range allKey { - allKey[i] = strings.Trim(allKey[i], emptyRunes) - - // 抽取 词表 @语句 集合key - // qid @ global_qid_whitelist - parts := strings.SplitN(allKey[i], "@", 2) - if len(parts) == 2 { - key_value := new(GroupKey) - if err := key_value.build("@", parts[0], parts[1]); err != nil { - return err - } - key_name := strings.Trim(parts[0], emptyRunes+"!") - this.keys[key_name] = key_value - continue - } - - // 抽取 小于 < 语句 范围 key - parts = strings.SplitN(allKey[i], "<", 2) - if len(parts) == 2 { - key_value := new(RangeKey) - if err := key_value.build("<", parts[0], parts[1]); err != nil { - return err - } - key_name := strings.Trim(parts[0], emptyRunes+"!") - this.keys[key_name] = key_value - continue - } - - // 抽取 小于 > 语句 范围key - parts = strings.SplitN(allKey[i], ">", 2) - if len(parts) == 2 { - key_value := new(RangeKey) - if err := key_value.build(">", parts[0], parts[1]); err != nil { - return err - } - key_name := strings.Trim(parts[0], emptyRunes+"!") - this.keys[key_name] = key_value - continue - } - - // 处理其他的 以 = 分割的语句，否则报错 - parts = strings.SplitN(allKey[i], "=", 2) - if len(parts) != 2 { - return errors.New("rule syntax error: keys error,miss sp =!") - } - if strings.ContainsAny(parts[1], "+-*") { - key_value := new(RangeKey) // 范围 - if err := key_value.build("=", parts[0], parts[1]); err != nil { - return err - } - key_name := strings.Trim(parts[0], emptyRunes+"!") - this.keys[key_name] = key_value - } else { - key_value := new(GroupKey) // 集合 - if err := key_value.build("=", parts[0], parts[1]); err != nil { - return err - } - key_name := strings.Trim(parts[0], emptyRunes+"!") - this.keys[key_name] = key_value - } - - } - return nil -} - -/************************************************************ - KoalaRule 使用过程，matche，相关方法 -************************************************************/ - -/** - * 缓存 key 拼装函数 - */ -func (this *KoalaRule) getCacheKey(gets map[string]string) string { - // cacheKey，先加上 r101 形式的前缀，代表所属规则，101等同于规则returnCode - cacheKey := "r" + strconv.Itoa(int(this.returnCode)) - - kForSort := make([]string, len(this.keys)) - i := 0 - for keyName, _ := range this.keys { - kForSort[i] = keyName - i++ - } - // 因map遍历时的顺序随机不定，但cache key必须确保一致； - // 故先对map的key字典序排序，后面按此顺序引用map值 - sort.Strings(kForSort) - - for _, keyName := range kForSort { - keyValue := this.keys[keyName] - switch keyValue.(type) { - case *GroupKey: - // combine的groupkey，不拼入，达到combine效果(联合计数) - if !keyValue.(*GroupKey).combine { - cacheKey = cacheKey + "|" + gets[keyName] - } - - default: - cacheKey = cacheKey + "|" + gets[keyName] - } - } - cacheKey = strings.Trim(cacheKey, "|") - return cacheKey -} - -/** - * 查询：查询规则当前的缓存值 - */ -func (this *KoalaRule) getCacheValue(cacheKey string) (int, error) { - redisConn := RedisPool.Get() - defer redisConn.Close() - - var err error - var cacheValue int - cacheValue, err = redis.Int(redisConn.Do("GET", cacheKey)) - if err == redis.ErrNil { - // 此 key 不存在，直接返回 0 - return 0, nil - } - if err != nil { - return 0, err - } - //println(cacheKey, " --> value:", cacheValue) - return cacheValue, nil -} - -/** - * 浏览；count规则缓存查询、比较 - */ -func (this *KoalaRule) countBrowse(cacheKey string) (bool, error) { - redisConn := RedisPool.Get() - defer redisConn.Close() - - var err error - var cacheValue int - cacheValue, err = redis.Int(redisConn.Do("GET", cacheKey)) - if err == redis.ErrNil { - // 此 key 不存在，直接返回通过 - return false, nil - } - if err != nil { - return false, err - } - //println(cacheKey, " --> value:", cacheValue) - if this.count == 0 || this.count > int32(cacheValue) { - return false, nil - } - return true, nil -} - -/** - * 更新；count规则缓存更新 - */ -func (this *KoalaRule) countUpdate(cacheKey string) error { - redisConn := RedisPool.Get() - defer redisConn.Close() - - var exists int - var err error - if exists, err = redis.Int(redisConn.Do("EXISTS", cacheKey)); err != nil { - return err - } - // set new cache - if exists == 0 { - var expiretime int32 - // 86400 按照 自然天计算 过期时间 - if this.time == 86400 { - y, m, d := time.Now().Date() - loc, _ := time.LoadLocation("Asia/Shanghai") - dayEnd := time.Date(y, m, d, 23, 59, 59, 0, loc).Unix() - expiretime = int32(dayEnd - time.Now().Unix()) - } else { - expiretime = this.time - } - - if _, err := redis.String(redisConn.Do("SETEX", cacheKey, expiretime, 1)); err != nil { - return err - } - - return nil - } - - // update - if _, err := redis.Int(redisConn.Do("INCR", cacheKey)); err != nil { - return err - } - - return nil -} - -/** - * 浏览；base方法缓存查询、比较 - */ -func (this *KoalaRule) baseBrowse(cacheKey string) (bool, error) { - redisConn := RedisPool.Get() - defer redisConn.Close() - - var err error - var cacheValue int - cacheValue, err = redis.Int(redisConn.Do("GET", cacheKey)) - if err == redis.ErrNil { - // 此 key 不存在，直接返回通过 - return false, nil - } - if err != nil { - return false, err - } - //println(cacheKey, " --> value:", cacheValue) - if this.base == 0 || this.base > int32(cacheValue) { - return false, nil - } - var cacheKey_time string = cacheKey + BASE_KEY_SUFFIX - - cacheValue, err = redis.Int(redisConn.Do("GET", cacheKey_time)) - if err == redis.ErrNil { - // 此 key 不存在，直接返回通过 - return false, nil - } - if err != nil { - return false, err - } - - //println(cacheKey_time, " --> value:", cacheValue) - if this.count == 0 || this.count > int32(cacheValue) { - return false, nil - } - return true, nil -} - -/** - * 更新；base方法缓存更新 - */ -func (this *KoalaRule) baseUpdate(cacheKey string) error { - redisConn := RedisPool.Get() - defer redisConn.Close() - - var exists int - var err error - if exists, err = redis.Int(redisConn.Do("EXISTS", cacheKey)); err != nil { - return err - } - if exists == 0 { - y, m, d := time.Now().Date() - dayEnd := time.Date(y, m, d, 23, 59, 59, 0, time.UTC).Unix() - expiretime := int32(dayEnd - time.Now().Unix()) - if _, err := redis.String(redisConn.Do("SETEX", cacheKey, expiretime, 1)); err != nil { - return err - } - - return nil - } - - // update - var cacheValue int - if cacheValue, err = redis.Int(redisConn.Do("INCR", cacheKey)); err != nil { - return err - } - if this.base == 0 || this.base > int32(cacheValue) { - return nil - } - - var cacheKey_time string = cacheKey + BASE_KEY_SUFFIX - if exists, err = redis.Int(redisConn.Do("EXISTS", cacheKey_time)); err != nil { - return err - } - if exists == 0 { - if _, err := redis.String(redisConn.Do("SETEX", cacheKey_time, this.time, 1)); err != nil { - return err - } - return nil - } - if _, err := redis.Int(redisConn.Do("INCR", cacheKey_time)); err != nil { - return err - } - return nil -} - -/** - * leak模式--查询 - * - */ -func (this *KoalaRule) leakBrowse(cacheKey string) (bool, error) { - redisConn := RedisPool.Get() - defer redisConn.Close() - - var err error - var listLen int - var edgeElement int64 - if listLen, err = redis.Int(redisConn.Do("LLEN", cacheKey)); err != nil { - return false, err - } - if listLen == 0 || listLen <= int(this.count) { - return false, nil - } - - defer func() { - go this.leakClear(cacheKey, listLen) - }() - - now := time.Now().Unix() - if edgeElement, err = redis.Int64(redisConn.Do("LINDEX", cacheKey, this.count)); err != nil { - return false, err - } - if int32(now-edgeElement) <= this.time { - return true, nil - } - return false, nil -} - -/** - * leak模式--清理 - * 清理队尾过期多余元素 - */ -func (this *KoalaRule) leakClear(cacheKey string, listLen int) { - redisConn := RedisPool.Get() - defer redisConn.Close() - - for listLen > int(this.count+1) { - if _, err := redis.Int64(redisConn.Do("RPOP", cacheKey)); err != nil { - return - } - listLen-- - } -} - -/** - * leak模式--更新 - */ -func (this *KoalaRule) leakUpdate(cacheKey string) error { - redisConn := RedisPool.Get() - defer redisConn.Close() - - now := time.Now().Unix() - if _, err := redis.Int(redisConn.Do("LPUSH", cacheKey, now)); err != nil { - return err - } - if _, err := redis.Int(redisConn.Do("EXPIRE", cacheKey, 2*this.time)); err != nil { - return err - } - return nil -} - -/** - * leak模式--反馈 - * 根据指令，减少桶内若干元素 - */ -func (this *KoalaRule) leakFeedback(cacheKey string, feedback int) error { - redisConn := RedisPool.Get() - defer redisConn.Close() - - for feedback > 0 { - if _, err := redis.Int64(redisConn.Do("LPOP", cacheKey)); err != nil { - return err - } - feedback-- - } - return nil -} - -/** - * 多重浏览；direct规则直接判定 - */ -func (this *KoalaRule) multiDirectBrowse(cacheKeys []interface{}) (map[string]bool, error) { - return nil, nil -} - -/** - * 多重浏览；count规则缓存查询、比较 - */ -func (this *KoalaRule) multiCountBrowse(cacheKeys []interface{}) (map[string]bool, error) { - redisConn := RedisPool.Get() - defer redisConn.Close() - - multiResult := make(map[string]bool, len(cacheKeys)) - cacheVals := make([]int, len(cacheKeys)) - intf := []interface{}{} - for i, _ := range cacheVals { - intf = append(intf, &cacheVals[i]) - } - reply, err := redis.Values(redisConn.Do("MGET", cacheKeys...)) - if err != nil { - return nil, err - } - if _, err := redis.Scan(reply, intf...); err != nil { - return nil, err - } - - for i, v := range cacheVals { - key := cacheKeys[i].(string) - if this.count == 0 || v == 0 || this.count > int32(v) { - multiResult[key] = false - continue - } - multiResult[key] = true - } - return multiResult, nil -} - -/** - * 多重浏览；base方法缓存查询、比较 - */ -func (this *KoalaRule) multiBaseBrowse(cacheKeys []interface{}) (map[string]bool, error) { - return nil, nil -} diff --git a/src/koala/main.go b/src/koala/main.go deleted file mode 100644 index 812af24..0000000 --- a/src/koala/main.go +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Koala Rule Engine Core - * - * @package: main - * @desc: koala engine - Main router - * - * koala频率控制服务 (规则引擎) - * 用途：是为了解决用户提交等相关频率控制的一个通用服务，主要是为了替换传统写死代码的频率控制模块以达到 高性能、灵活配置的要求。 - * 方案：支持高度灵活的规则配置；并实现了规则配置的动态加载； 后端cache采用带连接池的redis。 - * - * @author: heiyeluren - * @github: https://github.com/heiyeluren - * @blog: https://blog.csdn.net/heiyeshuwu - * - */ - -package main - -import ( - "github.com/garyburd/redigo/redis" - "runtime" - "utility/configs" - "utility/logger" -) - -var ( - Config *configs.Config // koala基本配置 - RedisPool *redis.Pool // 全局redis连接池 - - /** - * 全局 md5 值 - * 说明：定期检查 rule配置的变化，与此 md5 比较；实现动态更新规则 - */ - Policy_md5 string - /** - * 需要检查更新的文件列表 - * 范围：rule 文件 + dicts 文件 - * 赋值：每次reload规则时，将其中的 dicts 文件记录于此 - */ - DynamicUpdateFiles []string = []string{} -) - -func init() { - // 设置koala进程并发线程数 - runtime.GOMAXPROCS(runtime.NumCPU()) - // 初始化配置 - Config = newConfig() - // 初始化连接池 - initredisPool() -} - -/** - * koala服务进程的main函数 - */ -func main() { - // 保存进程的 pid 到文件中，供 stop、restart 脚本引用 - SavePid(Config.Get("pid_file")) - - // 初始化，并启动 logger 协程 - go logger.Log_Run(Config.GetAll()) - - // 首次加载规则 - var err error = PolicyInterpreter("") - if err != nil { - panic(err.Error()) - } - Policy_md5 = NewPolicyMD5() - - // 启动 rule统计协程 - go CounterAgent() - - // 启动 规则更新协程，定期检查 policy 更新 - go PolicyLoader() - - // 启动 http监听协程 - go FrontListen() - - // hold 住 main协程 - select {} -} diff --git a/src/koala/policy.go b/src/koala/policy.go deleted file mode 100644 index e764d7d..0000000 --- a/src/koala/policy.go +++ /dev/null @@ -1,266 +0,0 @@ -/** - * Koala Rule Engine Core - * - * @package: main - * @desc: koala engine - Dict list parser - * - * @author: heiyeluren - * @github: https://github.com/heiyeluren - * @blog: https://blog.csdn.net/heiyeshuwu - * - */ - -package main - -import ( - "encoding/json" - "errors" - "io/ioutil" - "strconv" - "strings" -) - -/** - * retValue数据类型 - */ -type RetValue struct { - Ret_type int32 - Ret_code int32 - Err_no int32 - Err_msg string - Str_reason string - Need_vcode int32 - Vcode_len int32 - Vcode_type int32 - Other string - Version int32 -} - -/** - * 策略结构，包含：dicts词表、rule规则、retValue 返回值三种数据; - * 其中 rule 是主体，dicts、和 retValue 会被 rule引用到 - */ -type Policy struct { - dictsTable map[string]map[string]string - ruleTable []KoalaRule - retValueTable map[int]RetValue -} - -/** - * 各种分隔符，trim的时候要除掉他们 - */ -const emptyRunes = " \r\t\v" - -var ( - GlobalPolicy *Policy // 全局策略配置 - TempPolicy *Policy //临时策略配置（用于策略的动态更新） -) - -/** - * Policy构造函数，完成各个元素的空间初始化 - */ -func NewPolicy() *Policy { - return &Policy{ - dictsTable: make(map[string]map[string]string), - ruleTable: make([]KoalaRule, 0, 50), - retValueTable: make(map[int]RetValue), - } -} - -/** - * Policy解释器，用于从文件解析配置，记录到 Policy 结构中 - */ -func PolicyInterpreter(extStream string) error { - // temp策略缓冲区初始化 - TempPolicy = NewPolicy() - - var rawStream []byte - var err error - if extStream != "" { - rawStream = []byte(extStream) - } else { - rawStream, err = ioutil.ReadFile(Config.Get("rule_file")) - if err != nil { - return errors.New("cannot load rule file") - } - } - - DynamicUpdateFiles = append(DynamicUpdateFiles, Config.Get("rule_file")) - lines := strings.Split(string(rawStream), "\n") - - // 配置文件分成三部分 词表、规则和返回结果，起始字符串分别是 [dicts] [rules] [result] - // 寻找这三部分的起始行 - dicts_pos, rules_pos, results_pos := 0, 0, 0 - for index, line := range lines { - line = strings.Trim(line, emptyRunes) - if line == "" || line[0] == '#' { - continue - } - if strings.EqualFold(strings.Trim(line, emptyRunes), "[dicts]") { - dicts_pos = index - } - if strings.EqualFold(strings.Trim(line, emptyRunes), "[rules]") { - rules_pos = index - } - if strings.EqualFold(strings.Trim(line, emptyRunes), "[result]") { - results_pos = index - } - } - - // 解析词表配置 - for index := dicts_pos + 1; index < rules_pos; index++ { - line := strings.Trim(lines[index], emptyRunes) - if line == "" || line[0] == '#' { - continue - } - if err := dictsBuilder(line); err != nil { - return errors.New(err.Error() + " ;AT-LINE-" + strconv.Itoa(index) + "; " + line) - } - } - - // 解析规则配置 - for index := rules_pos + 1; index < results_pos; index++ { - line := strings.Trim(lines[index], emptyRunes) - if line == "" || line[0] == '#' { - continue - } - if err := rulesBuilder(line); err != nil { - return errors.New(err.Error() + " ;AT-LINE-" + strconv.Itoa(index) + "; " + line) - } - //println(line) - } - - // 解析返回结果配置 - for index := results_pos + 1; index < len(lines); index++ { - line := strings.Trim(lines[index], emptyRunes) - if line == "" || line[0] == '#' { - continue - } - if err := resultsBuilder(line); err != nil { - return errors.New(err.Error() + " ;AT-LINE-" + strconv.Itoa(index) + "; " + line) - } - } - - // 校验规则有效性 - if err = ruleValidityCheck(); err != nil { - return err - } - - // 覆盖全局策略配置 - GlobalPolicy = TempPolicy - - return nil -} - -/** - * rule构造器，对单条 rule 进行解析 然后存入 TempPolicy - */ -func rulesBuilder(rule string) error { - // rule : [direct] [qid @ global_qid_whitelist] [time=1; count=0;] [result=1; return=101] - parts := strings.SplitN(rule, ":", 2) - if len(parts) == 2 && strings.EqualFold(strings.Trim(parts[0], emptyRunes), "rule") { - // - var singleRule KoalaRule - if err := singleRule.Constructor(parts[1]); err != nil { - return err - } - TempPolicy.ruleTable = append(TempPolicy.ruleTable, singleRule) - } else { - return errors.New("rule syntax error: struct error!") - } - return nil -} - -/** - * dicts构造器，对单条 dict 进行解析 然后存入 TempPolicy - */ -func dictsBuilder(dict string) error { - // 配置格式 名称 : 配置文件名 - // global_qid_whitelist : etc/global_qid_whitelist.dat - - parts := strings.SplitN(dict, ":", 2) - if len(parts) != 2 { - return errors.New("dict syntax error: struct error!") - } - oneDict := make(map[string]string, 10) - - // 读取配置文件 - fileName := strings.Trim(parts[1], emptyRunes) - DynamicUpdateFiles = append(DynamicUpdateFiles, fileName) - rawStream, err := ioutil.ReadFile(fileName) - if err != nil { - return errors.New("cannot load dict file") - } - lines := strings.Split(string(rawStream), "\n") - for _, v := range lines { - item := strings.Trim(v, emptyRunes) - oneDict[item] = item - } - dictName := strings.Trim(parts[0], emptyRunes) - TempPolicy.dictsTable[dictName] = oneDict - return nil -} - -/** - * retValue 构造器，对单条 result 进行解析 - */ -func resultsBuilder(result string) error { - // 1 : { "Ret_type":1, "Ret_code" : 0, "Err_no":0, "Err_msg":"", "Str_reason":"Allow", "Need_vcode":0, "Vcode_len":0, "Vcode_type":0, "Other":"", "Version":0 } - parts := strings.SplitN(result, ":", 2) - var retType int = -1 - var err error = nil - if retType, err = strconv.Atoi(strings.Trim(parts[0], emptyRunes)); err != nil { - return err - } - - var ret RetValue - inString := strings.Trim(parts[1], emptyRunes) - if err = json.Unmarshal([]byte(inString), &ret); err != nil { - return err - } - //fmt.Printf("%+v \n", ret) - TempPolicy.retValueTable[retType] = ret - return nil -} - -/** - * 功能：rule合法性检查（事后检查） - * 对象：TempPolicy - * 作用：此检查发生在，解析过程的末尾，对已经读入内存的配置，检查其合法性、逻辑正确性 - * a、return值 唯一性检查 - * b、base、count、time、result、return完整性检查 - * c、base、count、time、result、return的范围检查，如 0 值、负值等 - */ -func ruleValidityCheck() error { - var returnMap map[int32]string = make(map[int32]string) - - for _, singleRule := range TempPolicy.ruleTable { - switch singleRule.methord { - case "direct": - break - case "count": - if singleRule.count <= 0 || singleRule.time <= 0 { - return errors.New("rule semantic error: rule argument out of range!") - } - case "base": - if singleRule.base <= 0 || singleRule.count <= 0 || singleRule.time <= 0 { - return errors.New("rule semantic error: rule argument out of range!") - } - default: - } - - if singleRule.result <= 0 || singleRule.returnCode <= 0 { - return errors.New("rule semantic error: result invalid!") - } - - if _, OK := TempPolicy.retValueTable[int(singleRule.result)]; !OK { - return errors.New("rule semantic error: result type no found!") - } - - if _, OK := returnMap[singleRule.returnCode]; OK { - return errors.New("rule semantic error: rules with same return code!") - } - returnMap[singleRule.returnCode] = "hi" - } - return nil -} diff --git a/src/koala/policyCounter.go b/src/koala/policyCounter.go deleted file mode 100644 index b2cf36f..0000000 --- a/src/koala/policyCounter.go +++ /dev/null @@ -1,118 +0,0 @@ -/** - * Koala Rule Engine Core - * - * @package: main - * @desc: koala engine - Dict list parser & counter - * - * @author: heiyeluren - * @github: https://github.com/heiyeluren - * @blog: https://blog.csdn.net/heiyeshuwu - * - */ - -package main - -import ( - "fmt" - "time" - "utility/logger" -) - -type CountMessage struct { - rule_no int32 - decision int32 -} - -type Counter struct { - allow int64 - deny int64 -} - -const ( - ALLOW = 1 - DENY = 2 -) - -var ( - PolicyCounter map[string]map[int32]*Counter - CountTransChannal chan *CountMessage -) - -func init() { - PolicyCounter = make(map[string]map[int32]*Counter) - CountTransChannal = make(chan *CountMessage, 1024) -} - -// 从 channal 里获取策略结果，然后定期写入日志 -func CounterAgent() { - - for { - // 从channal获取一条countMsg（统计消息） - countMsg := <-CountTransChannal - - dateString := GetDateString(time.Now()) - - if _, OK := PolicyCounter[dateString]; !OK { - // 将过期的元素（上一时段的）转移存储，并从map摘除 - for k, v := range PolicyCounter { - recordExpiredCounter(v) - delete(PolicyCounter, k) - } - // 初始化当前时段map - PolicyCounter[dateString] = make(map[int32]*Counter) - } - if _, OK := PolicyCounter[dateString][countMsg.rule_no]; !OK { - // 如果map元素不存在，则初始化此元素 - initCounter := new(Counter) - PolicyCounter[dateString][countMsg.rule_no] = initCounter - } - - var singleCounter *Counter - singleCounter = PolicyCounter[dateString][countMsg.rule_no] - switch countMsg.decision { - case ALLOW: - singleCounter.allow += 1 - case DENY: - singleCounter.deny += 1 - default: - } - } -} - -/** - * 记录counter数据到日志 - */ -func recordExpiredCounter(counters map[int32]*Counter) { - logHandle := logger.NewLogger(time.Now().Format("20060102") + "_counter") - for k, v := range counters { - logMsg := fmt.Sprintf(" rule_no:%d allow:%d deny:%d ", k, v.allow, v.deny) - logHandle.Warning(logMsg) - } -} - -/** - * 统计API - */ -func CounterClient(ruleNo int32, deny bool) { - msg := new(CountMessage) - msg.rule_no = ruleNo - if deny == true { - msg.decision = DENY - } else { - msg.decision = ALLOW - } - - // 将统计消息，发给channal - CountTransChannal <- msg -} - -func GetDateString(t time.Time) string { - // 按天划分时段 - tString := t.Format("20060102") - return tString -} - -func GetCurrentCounters() map[int32]*Counter { - dateString := GetDateString(time.Now()) - return PolicyCounter[dateString] -} diff --git a/src/koala/policyLoader.go b/src/koala/policyLoader.go deleted file mode 100644 index e30cbdb..0000000 --- a/src/koala/policyLoader.go +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Koala Rule Engine Core - * - * @package: main - * @desc: koala engine - Dict list parser & loader - * - * @author: heiyeluren - * @github: https://github.com/heiyeluren - * @blog: https://blog.csdn.net/heiyeshuwu - * - */ - -package main - -import ( - "crypto/md5" - "errors" - "fmt" - "io" - "io/ioutil" - "time" - "utility/logger" -) - -/** - * Policy_md5 初始化函数 - */ -func NewPolicyMD5() string { - m, err := Policy_MD5() - if err != nil { - // error log - } - return m -} - -/** - * policy实时更新函数 - * 说明：用于实时更新rule配置，解析过程调用PolicyInterpreter()处理 - */ -func PolicyLoader() { - var err error - var m string - logHandle := logger.NewLogger("") - - for { - d := Config.GetInt("policy_loader_frequency") - if d == 0 { - d = 300 - } - time.Sleep(time.Duration(d) * time.Second) - - m, err = Policy_MD5() - if err != nil { - logHandle.Warning("[errmsg=" + err.Error() + " md5=" + Policy_md5 + "]") - } - //println(Policy_md5) - - if m != Policy_md5 { - DynamicUpdateFiles = []string{} - err = PolicyInterpreter("") - if err != nil { - logHandle.Warning("[errmsg=" + err.Error() + "]") - } else { - Policy_md5 = m - logHandle.Trace("[msg=policy reload! new-md5=" + Policy_md5 + "]") - } - } - } -} - -/** - * 计算 DynamicUpdateFiles 所包含文件的 md5 值 - */ -func Policy_MD5() (string, error) { - contentStream := "" - for _, file := range DynamicUpdateFiles { - rawStream, err := ioutil.ReadFile(file) - if err != nil { - return "", errors.New("cannot load policy file!") - } - contentStream += string(rawStream) - } - - files_MD5 := md5.New() - io.WriteString(files_MD5, contentStream) - - string_md5 := fmt.Sprintf("%x", files_MD5.Sum(nil)) - return string_md5, nil -} diff --git a/src/koala/service.go b/src/koala/service.go deleted file mode 100644 index c79cdea..0000000 --- a/src/koala/service.go +++ /dev/null @@ -1,493 +0,0 @@ -/** - * Koala Rule Engine Core - * - * @package: main - * @desc: koala engine - Http api access process - * - * @author: heiyeluren - * @github: https://github.com/heiyeluren - * @blog: https://blog.csdn.net/heiyeshuwu - * - */ - -package main - -import ( - "encoding/json" - "net/url" - "strconv" - "strings" - "utility/logger" - "utility/network" -) - -/** - * 查询访问接口 - */ -func (this *FrontServer) DoRuleBrowse(request *network.HttpRequest, response *network.HttpResponse, logHandle *logger.Logger) { - // 本地策略指针，可避免匹配过程中Global策略被替换 导致不一致 - var localPolicy *Policy = GlobalPolicy - - var singleRule KoalaRule - var err error - var retValue RetValue = localPolicy.retValueTable[0] - // 匹配每一条rule规则 - for _, singleRule = range localPolicy.ruleTable { - var satisfied bool = true - // 遍历每个key，若不匹配或者参数未传，不命中 - for k, v := range singleRule.keys { - s := request.Gstr(k) - if s != "" && v.matches(s) { - continue - } - satisfied = false - break - } - if !satisfied { - continue - } - - // 对命中的key，查缓存值，与阀值比较，判断是否超出限制 - var isOut bool - ruleCacheKey := singleRule.getCacheKey(request.Gets()) - //println(ruleCacheKey) - switch singleRule.methord { - case "direct": - isOut = true - case "count": - if isOut, err = singleRule.countBrowse(ruleCacheKey); err != nil { - logHandle.Fatal("[errmsg=" + err.Error() + "]") - } - case "base": - if isOut, err = singleRule.baseBrowse(ruleCacheKey); err != nil { - logHandle.Fatal("[errmsg=" + err.Error() + "]") - } - case "leak": - if isOut, err = singleRule.leakBrowse(ruleCacheKey); err != nil { - logHandle.Fatal("[errmsg=" + err.Error() + "]") - } - default: - } - - // 统计，记录策略判定数据 - CounterClient(singleRule.returnCode, isOut) - - // 超出限制，按照rule的约定，给出处置策略 - if isOut { - retValue = localPolicy.retValueTable[int(singleRule.result)] - retValue.Ret_code = singleRule.returnCode - break - } - retValue = localPolicy.retValueTable[1] - } - - // _writeThrough“直接写缓存”开关，同时完成 Browse和 Update两步操作。 - if retValue.Ret_type >= 1 && request.Gstr("_writeThrough") == "yes" { - go RuleUpdateLogic(request, logHandle) - } - - // 返回json结果 - var retString []byte - retString, err = json.Marshal(retValue) - if err != nil { - response.SetCode(500) - return - } - response.Puts(string(retString)) - - response.SetCode(200) -} - -/** - * 更新访问接口 - */ -func (this *FrontServer) DoRuleUpdate(request *network.HttpRequest, response *network.HttpResponse, logHandle *logger.Logger) { - go RuleUpdateLogic(request, logHandle) - - response.Puts(`{"err_no":0, "err_msg":"OK"}`) - response.SetCode(200) -} - -/** - * 查询缓存状态接口 - */ -/* -func (this *FrontServer) DoRuleValue(request *network.HttpRequest, response *network.HttpResponse, logHandle *logger.Logger) { - var localPolicy *Policy = GlobalPolicy - - var singleRule KoalaRule - var err error - var cacheVal int = -1 - var req_rule int32 = int32(request.Gint("rule_no")) - for _, singleRule = range localPolicy.ruleTable { - if singleRule.returnCode != req_rule { - continue - } - ruleCacheKey := singleRule.getCacheKey(request.Gets()) - if cacheVal, err = singleRule.getCacheValue(ruleCacheKey); err != nil { - logHandle.Fatal("[errmsg=" + err.Error() + "]") - response.Puts(`{"err_no":-1, "err_msg":"system error!"}`) - response.SetCode(500) - break - } - } - if cacheVal == -1 { - response.Puts(`{"err_no":-2, "err_msg":"unknown rule_no!"}`) - response.SetCode(400) - } else { - response.Puts(`{"err_no":0, "err_msg":"", "cache_val":` + strconv.Itoa(cacheVal) + `}`) - response.SetCode(200) - } -} -*/ - -/** - * 非中断查询接口（可命中、并返回多条策略） - */ -func (this *FrontServer) DoRuleBrowse_complete(request *network.HttpRequest, response *network.HttpResponse, logHandle *logger.Logger) { - // 本地策略指针，可避免匹配过程中Global策略被替换 导致不一致 - var localPolicy *Policy = GlobalPolicy - - var singleRule KoalaRule - var err error - // 用于返回多个结果，RetValue数组 - var retArray []RetValue - var retValue RetValue = localPolicy.retValueTable[0] - // 匹配每一条rule规则 - for _, singleRule = range localPolicy.ruleTable { - var satisfied bool = true - // 遍历每个key，若不匹配或者参数未传，跳过 - for k, v := range singleRule.keys { - s := request.Gstr(k) - if s != "" && v.matches(s) { - continue - } - satisfied = false - break - } - if !satisfied { - continue - } - - // 对匹配的key，查缓存值，与阀值比较，判断是否超出限制 - var isOut bool - switch singleRule.methord { - case "direct": - isOut = true - case "count": - ruleCacheKey := singleRule.getCacheKey(request.Gets()) - if isOut, err = singleRule.countBrowse(ruleCacheKey); err != nil { - logHandle.Fatal("[errmsg=" + err.Error() + "]") - } - case "base": - ruleCacheKey := singleRule.getCacheKey(request.Gets()) - if isOut, err = singleRule.baseBrowse(ruleCacheKey); err != nil { - logHandle.Fatal("[errmsg=" + err.Error() + "]") - } - default: - } - - // 统计，记录策略判定数据 - CounterClient(singleRule.returnCode, isOut) - - // 命中，拼装结果 - if isOut { - retValue = localPolicy.retValueTable[int(singleRule.result)] - retValue.Ret_code = singleRule.returnCode - retArray = append(retArray, retValue) - } - } - - // 如果没有命中任何策略，返回默认值 - if retValue.Ret_type == 0 { - retArray = append(retArray, retValue) - } - - // _writeThrough“直接写缓存”开关，同时完成 Browse和 Update两步操作。 - if retValue.Ret_type <= 1 && request.Gstr("_writeThrough") == "yes" { - RuleUpdateLogic(request, logHandle) - } - - // 返回json结果 - var retString []byte - retString, err = json.Marshal(retArray) - if err != nil { - response.SetCode(500) - return - } - response.Puts(string(retString)) - - response.SetCode(200) -} - -/** - * 反馈-feedback-接口 - */ -/* -func (this *FrontServer) DoRuleFeedback(request *network.HttpRequest, response *network.HttpResponse, logHandle *logger.Logger) { - - var localPolicy *Policy = GlobalPolicy - - var singleRule KoalaRule - var feedbackType string = request.Gstr("_feedbackType") - // 匹配每一条rule规则 - for _, singleRule = range localPolicy.ruleTable { - var satisfied bool = true - // 遍历每个key，若不匹配或者参数未传，不命中 - for k, v := range singleRule.keys { - s := request.Gstr(k) - if s != "" && v.matches(s) { - continue - } - satisfied = false - break - } - if !satisfied { - continue - } - - var feedback int32 - switch feedbackType { - case "erase1": - feedback = singleRule.erase1 - case "erase2": - feedback = singleRule.erase2 - default: - feedback = singleRule.erase1 - } - - ruleCacheKey := singleRule.getCacheKey(request.Gets()) - go singleRule.leakFeedback(ruleCacheKey, int(feedback)) - } - - response.Puts(`{"err_no":0, "err_msg":"OK"}`) - response.SetCode(200) -} -*/ - -/** - * 更新操作执行函数 - */ -func RuleUpdateLogic(request *network.HttpRequest, logHandle *logger.Logger) { - // 本地策略指针，可避免匹配过程中Global策略被替换 导致不一致 - var localPolicy *Policy = GlobalPolicy - - // 匹配每一条rule规则 - var singleRule KoalaRule - for _, singleRule = range localPolicy.ruleTable { - var satisfied bool = true - // 遍历每个key，若不匹配或者参数未传，不命中 - for k, v := range singleRule.keys { - s := request.Gstr(k) - if s != "" && v.matches(s) { - continue - } - satisfied = false - break - } - if !satisfied { - continue - } - var ruleCacheKey string - var err error - - ruleCacheKey = singleRule.getCacheKey(request.Gets()) - // 更新cache值 - switch singleRule.methord { - case "count": - if singleRule.count == 0 { - continue - } - if err = singleRule.countUpdate(ruleCacheKey); err != nil { - logHandle.Fatal("[errmsg=" + err.Error() + "]") - continue - } - case "base": - if err = singleRule.baseUpdate(ruleCacheKey); err != nil { - logHandle.Fatal("[errmsg=" + err.Error() + "]") - continue - } - case "leak": - if err = singleRule.leakUpdate(ruleCacheKey); err != nil { - logHandle.Fatal("[errmsg=" + err.Error() + "]") - continue - } - default: - } - } -} - -/** - * 监控连接redis是否成功 - */ -func (this *FrontServer) DoMonitorAlive(request *network.HttpRequest, response *network.HttpResponse, logHandle *logger.Logger) { - redisConn := RedisPool.Get() - defer redisConn.Close() - _, err := redisConn.Do("PING") - if err != nil { - response.Puts(`{"errno": -1, "errmsg": "redis-error!"}`) - response.SetCode(500) - logHandle.Fatal("[errmsg=" + err.Error() + "]") - return - } - response.Puts(`{"errno": 0, "errmsg": "OK!"}`) - response.SetCode(200) -} - -type Job struct { - Id string - Arg string -} - -type JobResult struct { - Id string - Result RetValue -} - -type JobBuffer struct { - Id string - args map[string]string - key string - status bool - decision int - retCode int32 -} - -/** - * 多重浏览访问接口 - */ -func (this *FrontServer) DoMultiBrowse(request *network.HttpRequest, response *network.HttpResponse, logHandle *logger.Logger) { - argsJson := request.Gstr("argsJson") - if argsJson == "" { - response.SetCode(400) - return - } - - var err error - var jobs []Job - if err = json.Unmarshal([]byte(argsJson), &jobs); err != nil { - logHandle.Fatal("[errmsg=" + err.Error() + "]") - response.SetCode(400) - return - } - - var logMsg string = "" - logMsg += "[ cip=" + request.GetRemoteIP() - logMsg += " intf=" + request.PathInfo() - - var buffers []JobBuffer - //var job Job - for _, job := range jobs { - var buf JobBuffer - buf.Id = job.Id - buf.args = parseJobArgs(job.Arg) - buf.status = false - buf.decision = 0 - buffers = append(buffers, buf) - logMsg += " ID" + job.Id + "@" + job.Arg - } - logMsg += " ] [" - - var localPolicy *Policy = GlobalPolicy - var singleRule KoalaRule - for _, singleRule = range localPolicy.ruleTable { - var cacheKeys []interface{} - for i, buf := range buffers { - buffers[i].key = "" - var satisfied bool = true - for k, v := range singleRule.keys { - s, OK := buf.args[k] - if OK && v.matches(s) { - continue - } - satisfied = false - break - } - if satisfied == true && buf.status == false { - buffers[i].key = singleRule.getCacheKey(buf.args) - cacheKeys = append(cacheKeys, buffers[i].key) - } - } - - if len(cacheKeys) == 0 { - continue - } - - var multiResult map[string]bool - switch singleRule.methord { - case "direct": - if multiResult, err = singleRule.multiDirectBrowse(cacheKeys); err != nil { - // err log - } - case "count": - if multiResult, err = singleRule.multiCountBrowse(cacheKeys); err != nil { - logHandle.Fatal("[errmsg=" + err.Error() + "]") - } - case "base": - if multiResult, err = singleRule.multiBaseBrowse(cacheKeys); err != nil { - // err log - } - default: - } - - // 统计，记录策略判定数据 - for _, decision := range multiResult { - CounterClient(singleRule.returnCode, decision) - } - - for i, buf := range buffers { - isOut, OK := multiResult[buf.key] - if OK && isOut { - buffers[i].status = true - buffers[i].decision = int(singleRule.result) - buffers[i].retCode = singleRule.returnCode - } else if OK && buf.status == false { - buffers[i].decision = 1 - } - } - } - - var jobResults []JobResult - for _, buf := range buffers { - var singleResult JobResult - singleResult.Id = buf.Id - singleResult.Result = localPolicy.retValueTable[buf.decision] - singleResult.Result.Ret_code = buf.retCode - jobResults = append(jobResults, singleResult) - logMsg += " ID" + buf.Id + "~Ret_code:" + strconv.Itoa(int(buf.retCode)) - } - logMsg += " ]" - logHandle.Notice(logMsg) - - // 返回json结果 - var retString []byte - retString, err = json.Marshal(jobResults) - if err != nil { - response.SetCode(500) - return - } - response.Puts(string(retString)) - - response.SetCode(200) - -} - -/** - * - */ -func parseJobArgs(rawArg string) map[string]string { - retMap := make(map[string]string, 0) - argString, err := url.QueryUnescape(rawArg) - if err != nil { - return nil - } - kvStrings := strings.Split(string(argString), "&") - for _, kvString := range kvStrings { - parts := strings.SplitN(kvString, "=", 2) - if len(parts) != 2 { - return nil - } - retMap[parts[0]] = parts[1] - } - return retMap -} diff --git a/src/koala/util.go b/src/koala/util.go deleted file mode 100644 index b3dd5cd..0000000 --- a/src/koala/util.go +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Koala Rule Engine Core - * - * @package: main - * @desc: koala engine - Utils functions - * - * @author: heiyeluren - * @github: https://github.com/heiyeluren - * @blog: https://blog.csdn.net/heiyeshuwu - * - */ - -package main - -import ( - "errors" - "io/ioutil" - "os" - "strconv" - "strings" -) - -/** - * 进程 pid 记录函数 - */ -func SavePid(pidFile string) { - pid := os.Getpid() - pidString := strconv.Itoa(pid) - ioutil.WriteFile(pidFile, []byte(pidString), 0777) -} - -/** - * 判断字符串是否是 IP 地址 - */ -func IsIPAddress(s string) bool { - // 符合x.x.x.x模式的字符串，即认为是ip地址 - //println(s) - parts := strings.Split(s, ".") - if len(parts) == 4 { - return true - } - return false -} - -/** - * 将代表ip地址、纯数字的string值 都统一转换成int64值 - */ -func ToInteger64(s string) (int64, error) { - var err error = nil - var result int64 = 0 - if IsIPAddress(s) { - parts := strings.Split(s, ".") - for i, v := range parts { - var intVal int - if intVal, err = strconv.Atoi(v); err != nil { - return result, err - } - if intVal > 255 || intVal < 0 { - return result, errors.New("rule syntax error: scope error,out of ip range 0-255!") - } - result += int64(intVal) << uint((3-i)*8) - } - } else { - var int64Val int64 - if int64Val, err = strconv.ParseInt(s, 10, 64); err != nil { - return 0, err - } - result = int64Val - } - return result, nil -} diff --git a/src/utility/configs/configs.go b/src/utility/configs/configs.go deleted file mode 100644 index 2e9de48..0000000 --- a/src/utility/configs/configs.go +++ /dev/null @@ -1,132 +0,0 @@ -/** - * Koala Rule Engine Core - * - * @package: main - * @desc: koala engine - config file parse engine - * - * @author: heiyeluren - * @github: https://github.com/heiyeluren - * @blog: https://blog.csdn.net/heiyeshuwu - * - */ - -package configs - -import ( - "errors" - "io/ioutil" - "path" - "strconv" - "strings" -) - -type Config struct { - data map[string]string -} - -func NewConfig() *Config { - return &Config{data: make(map[string]string)} -} - -const emptyRunes = " \r\t\v" - -func (this *Config) Load(configFile string) error { - stream, err := ioutil.ReadFile(configFile) - if err != nil { - return errors.New("cannot load config file") - } - content := string(stream) - lines := strings.Split(content, "\n") - for _, line := range lines { - line = strings.Trim(line, emptyRunes) - // 去除空行和注释 - if line == "" || line[0] == '#' { - continue - } - parts := strings.SplitN(line, "=", 2) - if len(parts) == 2 { - for i, part := range parts { - parts[i] = strings.Trim(part, emptyRunes) - } - this.data[parts[0]] = parts[1] - } else { - // 判断并处理include条目，load相应的config文件 - // include 的配置文件应该在当前配置文件所在的目录下 - includes := strings.SplitN(parts[0], " ", 2) - if len(includes) == 2 && strings.EqualFold(includes[0], "include") { - // 拼解新包含config文件的path - confDir := path.Dir(configFile) - newConfName := strings.Trim(includes[1], emptyRunes) - newConfPath := path.Join(confDir, newConfName) - // 载入include的config文件，调用Load自身 - err := this.Load(newConfPath) - if err != nil { - return errors.New("load include config file failed") - } - continue - } else { - return errors.New("invalid config file syntax") - } - } - } - return nil -} - -func (this *Config) GetAll() map[string]string { - return this.data -} - -func (this *Config) Get(key string) string { - if value, ok := this.data[key]; ok { - return value - } - return "" -} - -func (this *Config) GetInt(key string) int { - value := this.Get(key) - if value == "" { - return 0 - } - result, err := strconv.Atoi(value) - if err != nil { - return 0 - } - return result -} - -func (this *Config) GetInt64(key string) int64 { - value := this.Get(key) - if value == "" { - return 0 - } - result, err := strconv.ParseInt(value, 10, 64) - if err != nil { - return 0 - } - return result -} - -func (this *Config) GetSlice(key string, separator string) []string { - slice := []string{} - value := this.Get(key) - if value != "" { - for _, part := range strings.Split(value, separator) { - slice = append(slice, strings.Trim(part, emptyRunes)) - } - } - return slice -} - -func (this *Config) GetSliceInt(key string, separator string) []int { - slice := this.GetSlice(key, separator) - results := []int{} - for _, part := range slice { - result, err := strconv.Atoi(part) - if err != nil { - continue - } - results = append(results, result) - } - return results -} diff --git a/src/utility/logger/log.go b/src/utility/logger/log.go deleted file mode 100644 index e5d74a9..0000000 --- a/src/utility/logger/log.go +++ /dev/null @@ -1,899 +0,0 @@ -/** - * @file: log.go - * @package: main - * @author: heiyeluren - * @desc: Log operate file - * @date: 2013/6/24 - * @history: - * 2013/6/24 created file - * 2013/7/1 add logid function - * 2013/7/2 update code structure - * 2013/7/4 refactor all code - * 2013/7/10 add log_level operate - */ - -package logger - -import ( - "errors" - "fmt" - "math/rand" - "os" - "runtime" - "strconv" - "strings" - "sync" - "time" -) - -/* - -配置文件格式： -================================================= -#日志文件位置 (例：/var/log/koala.log) -log_notice_file_path = log/koala.log -log_debug_file_path = log/koala.log -log_trace_file_path = log/koala.log -log_fatal_file_path = log/koala.log.wf -log_warning_file_path = log/koala.log.wf - -#日志文件切割周期（1天:day; 1小时:hour; 10分钟:ten） -log_cron_time = day - -#日志chan队列的buffer长度，建议不要少于1024，#不建议多于102400，最长：2147483648 -log_chan_buff_size = 20480 - -#日志刷盘的间隔时间，单位:毫秒，建议500~5000毫秒(0.5s-5s)，建议不超过30秒 -log_flush_timer = 1000 -================================================= - -代码调用示例： - -import "utility/logger" -import "utility/network" - -logid := request.Header("WD_REQUEST_ID") //注意:只有问答产品才有WD_REQUEST_ID这个数据，其他服务按照对应id来 -log = logger.NewLogger(logid) //注意: 一个请求只能New一次，logid可以传空字符串，则会内部自己生成logid - -log_notice_msg := "[clientip=202.106.51.6 errno=0 errmsg=ok request_time=100ms]" -log.Notice(log_notice_msg) - -log_warning_msg := "[clientip=202.106.51.6 errno=101 errmsg=\"client is error\" request_time=45ms]" -log.Warning(log_warning_msg) - -*/ - -//======================== -// -// 外部调用Logger方法 -// -//======================== - -/** - * Log 每次请求结构体数据 - */ -type Logger struct { - Logid string -} - -//日志级别类型常量 -const ( - LOG_TYPE_FATAL = 1 - LOG_TYPE_WARNING = 2 - LOG_TYPE_NOTICE = 4 - LOG_TYPE_TRACE = 8 - LOG_TYPE_DEBUG = 16 -) - -//日志类型对应信息 -const ( - LOG_TYPE_FATAL_STR = "FATAL" - LOG_TYPE_WARNING_STR = "WARNING" - LOG_TYPE_NOTICE_STR = "NOTICE" - LOG_TYPE_TRACE_STR = "TRACE" - LOG_TYPE_DEBUG_STR = "DEBUG" -) - -//日志信息map -var G_Log_Type_Map map[int]string = map[int]string{ - LOG_TYPE_FATAL: LOG_TYPE_FATAL_STR, - LOG_TYPE_WARNING: LOG_TYPE_WARNING_STR, - LOG_TYPE_NOTICE: LOG_TYPE_NOTICE_STR, - LOG_TYPE_TRACE: LOG_TYPE_TRACE_STR, - LOG_TYPE_DEBUG: LOG_TYPE_DEBUG_STR, -} - -//------------------------ -// logger外部调用方法 -//------------------------ - -/** - * 构造函数 - * - */ -func NewLogger(logid string) *Logger { - return &Logger{Logid: logid} -} - -/** - * 正常请求日志打印调用 - * - * 注意： - * 每个请求(request)只能调用本函数一次，函数里必须携带必须字段: ip, errno, errmsg 等字段，其他kv信息自己组织 - * - * 示例： - * Log_Notice("clientip=192.168.0.1 errno=0 errmsg=ok key1=valu2 key2=valu2") - * - */ -func (l *Logger) Notice(log_messgae string) { - l.sync_msg(LOG_TYPE_NOTICE, log_messgae) -} - -/** - * 函数调用栈trace日志打印调用 - * - */ -func (l *Logger) Trace(log_messgae string) { - l.sync_msg(LOG_TYPE_TRACE, log_messgae) -} - -/** - * 函数调用调试debug日志打印调用 - * - */ -func (l *Logger) Debug(log_messgae string) { - l.sync_msg(LOG_TYPE_DEBUG, log_messgae) -} - -/** - * 致命错误Fatal日志打印调用 - * - */ -func (l *Logger) Fatal(log_messgae string) { - l.sync_msg(LOG_TYPE_FATAL, log_messgae) -} - -/** - * 警告错误warging日志打印调用 - * - */ -func (l *Logger) Warning(log_messgae string) { - l.sync_msg(LOG_TYPE_WARNING, log_messgae) -} - -//------------------------ -// logger内部使用方法 -//------------------------ - -/** - * 写入日志到channel - * - */ -func (l *Logger) sync_msg(log_type int, log_msg string) error { - //init request log - //Log_New() - - //从配置日志级别log_level判断当前日志是否需要入channel队列 - if (log_type & G_Log_V.LogLevel) != log_type { - return nil - } - - //G_Log_V := Log_New(G_Log_V) - if log_type <= 0 || log_msg == "" { - errors.New("log_type or log_msg param is empty") - } - - //拼装消息内容 - log_str := l.pad_msg(log_type, log_msg) - - //日志类型 - if _, ok := G_Log_Type_Map[log_type]; !ok { - errors.New("log_type is invalid") - } - - //设定消息格式 - log_msg_data := Log_Msg_T{ - LogType: log_type, - LogData: log_str, - } - - //写消息到channel - G_Log_V.LogChan <- log_msg_data - - //判断当前整个channel 的buffer大小是否超过90%的阀值，超过就直接发送刷盘信号 - var threshold float32 - var curr_chan_len int = len(G_Log_V.LogChan) - threshold = float32(curr_chan_len) / float32(G_Log_V.LogChanBuffSize) - - if threshold >= 0.9 && G_Flush_Log_Flag != true { - G_Flush_Lock.Lock() - G_Flush_Log_Flag = true - G_Flush_Lock.Unlock() - - G_Log_V.FlushLogChan <- true - //打印目前达到阀值了 - if Log_Is_Debug() { - Log_Debug_Print(fmt.Sprintf("Out threshold!! Current G_Log_V.LogChan: %v; G_Log_V.LogChanBuffSize: %v", curr_chan_len, G_Log_V.LogChanBuffSize), nil) - } - } - - return nil -} - -/** - * 拼装日志消息 - * - * 说明： - * 主要是按照格式把消息给拼装起来 - * - * 日志格式示例： - * NOTICE: 2013-06-28 18:30:56 koala [logid=1234 filename=yyy.go lineno=29] [clientip=10.5.0.108 errno=0 errmsg="ok"] - * WARNING: 2013-06-28 18:30:56 koala [logid=1234 filename=yyy.go lineno=29] [clientip=10.5.0.108 errno=404 errmsg="json format invalid"] - */ -func (l *Logger) pad_msg(log_type int, log_msg string) string { - - var ( - //日志拼装格式字符串 - log_format_str string - log_ret_str string - - //日志所需字段变量 - log_type_str string - log_date_time string - log_id string - log_filename string - log_lineno int - log_callfunc string - - //log_clientip string - //log_errno int - //log_errmsg string - - //其他变量 - ok bool - fcName uintptr - ) - - //获取调用的 函数/文件名/行号 等信息 - fcName, log_filename, log_lineno, ok = runtime.Caller(3) - if !ok { - errors.New("call runtime.Caller() fail") - } - log_callfunc = runtime.FuncForPC(fcName).Name() - - //展现调用文件名最后两段 - //println(log_filename) - - //判断当前操作系统路径分割符，获取调用文件最后两组路径信息 - os_path_separator := Log_Get_Os_Separator(log_filename) - call_path := strings.Split(log_filename, os_path_separator) - if path_len := len(call_path); path_len > 2 { - log_filename = strings.Join(call_path[path_len-2:], os_path_separator) - } - - //获取当前日期时间 (#吐槽: 不带这么奇葩的调用参数好不啦！难道这天是Go诞生滴日子??!!!#) - log_date_time = time.Now().Format("2006-01-02 15:04:05") - - //app name - //log_app_name = "koala" - - //logid读取 - log_id = l.get_logid() - - //日志类型 - if log_type_str, ok = G_Log_Type_Map[log_type]; !ok { - errors.New("log_type is invalid") - } - - //拼装返回 - log_format_str = "%s: %s [logid=%s file=%s no=%d call=%s] %s\n" - log_ret_str = fmt.Sprintf(log_format_str, log_type_str, log_date_time, log_id, log_filename, log_lineno, log_callfunc, log_msg) - - //调试 - //println(log_ret_str) - - return log_ret_str -} - -/** - * 获取LogID - * - * 说明： - * 从客户端request http头里看看是否可以获得logid，http头里可以传递一个：WD_REQUEST_ID - * 如果没有传递，则自己生成唯一logid - */ -func (l *Logger) get_logid() string { - //获取request http头中的logid字段 - if l.Logid != "" { - return l.Logid - } - return l.gen_logid() -} - -/** - * 生成当前请求的Log ID - * - * 策略： - * 主要是保证唯一logid，采用当前纳秒级时间+随机数生成 - */ -func (l *Logger) gen_logid() string { - //获取当前时间 - microtime := time.Now().UnixNano() - - //生成随机数 - r := rand.New(rand.NewSource(microtime)) - randNum := r.Intn(100000) - - //生成logid：把纳秒时间+随机数生成 (注意：int64的转string使用 FormatInt，int型使用Itoa就行了) - //logid := fmt.Sprintf("%d%d", microtime, randNum) - logid := strconv.FormatInt(microtime, 10) + strconv.Itoa(randNum) - - return logid -} - -//======================== -// -// 内部协程Run函数 -// -//======================== - -/** - * 单条日志结构 - */ -type Log_Msg_T struct { - LogType int - LogData string -} - -/** - * Log主chan队列配置 - */ -type Log_T struct { - - //------------------ - // Channel数据 - //------------------ - - //日志接收channel队列 - LogChan chan Log_Msg_T - - //是否马上日志刷盘: true or false，如果为true，则马上日志刷盘 (本chan暂时没有使用) - FlushLogChan chan bool - - //------------------ - // 配置相关数据 - //------------------ - - //所有日志文件位置 - LogFilePath map[int]string - - //日志文件位置 (例：/var/log/koala.log 和 /var/log/koala.log.wf) - LogNoticeFilePath string - LogErrorFilePath string - - //写入日志切割周期（1天:day、1小时:hour、15分钟：Fifteen、10分钟：Ten） - LogCronTime string - - //日志chan队列的buffer长度，建议不要少于1024，不多于102400，最长：2147483648 - LogChanBuffSize int - - //按照间隔时间日志刷盘的日志的间隔时间，单位：秒，建议1~5秒，不超过256 - LogFlushTimer int - - //------------------ - // 运行时相关数据 - //------------------ - - //去重的日志文件名和fd (实际需需要物理写入文件名和句柄) - MergeLogFile map[string]string - MergeLogFd map[string]*os.File - - //上游配置的map数据(必须包含所有所需项) - RunConfigMap map[string]string - - //是否开启日志库调试模式 - LogDebugOpen bool - - //日志打印的级别（需要打印那些日志） - LogLevel int - - // 日志文件的存在时间, 单位:天 - LogLifeTime int -} - -/** - * 配置项相关常量&变量 - */ -const ( - LOG_CONF_NOTICE_FILE_PATH = "log_notice_file_path" - LOG_CONF_DEBUG_FILE_PATH = "log_debug_file_path" - LOG_CONF_TRACE_FILE_PATH = "log_trace_file_path" - LOG_CONF_FATAL_FILE_PATH = "log_fatal_file_path" - LOG_CONF_WARNING_FILE_PATH = "log_warning_file_path" - - LOG_CONF_CRON_TIME = "log_cron_time" - LOG_CONF_CHAN_BUFFSIZE = "log_chan_buff_size" - LOG_CONF_FLUSH_TIMER = "log_flush_timer" - LOG_CONF_DEBUG_OPEN = "log_debug_open" - LOG_CONF_LEVEL = "log_level" - LOG_CONF_FILE_LIFE_TIME = "log_file_ilfe_time" -) - -//配置选项值类型(字符串或数字) -const ( - LOG_CONF_TYPE_STR = 1 - LOG_CONF_TYPE_NUM = 2 -) - -//配置项map全局变量 (定义一个选项输入的值是字符串还是数字) -var G_Conf_Item_Map map[string]int = map[string]int{ - LOG_CONF_NOTICE_FILE_PATH: LOG_CONF_TYPE_STR, - LOG_CONF_DEBUG_FILE_PATH: LOG_CONF_TYPE_STR, - LOG_CONF_TRACE_FILE_PATH: LOG_CONF_TYPE_STR, - LOG_CONF_FATAL_FILE_PATH: LOG_CONF_TYPE_STR, - LOG_CONF_WARNING_FILE_PATH: LOG_CONF_TYPE_STR, - - LOG_CONF_CRON_TIME: LOG_CONF_TYPE_STR, - LOG_CONF_CHAN_BUFFSIZE: LOG_CONF_TYPE_NUM, - LOG_CONF_FLUSH_TIMER: LOG_CONF_TYPE_NUM, - LOG_CONF_DEBUG_OPEN: LOG_CONF_TYPE_NUM, - LOG_CONF_LEVEL: LOG_CONF_TYPE_NUM, - LOG_CONF_FILE_LIFE_TIME: LOG_CONF_TYPE_NUM, -} - -//日志文件名与日志类型的映射 -var G_Conf_FileToType_Map map[string]int = map[string]int{ - LOG_CONF_NOTICE_FILE_PATH: LOG_TYPE_NOTICE, - LOG_CONF_DEBUG_FILE_PATH: LOG_TYPE_DEBUG, - LOG_CONF_TRACE_FILE_PATH: LOG_TYPE_TRACE, - LOG_CONF_FATAL_FILE_PATH: LOG_TYPE_FATAL, - LOG_CONF_WARNING_FILE_PATH: LOG_TYPE_WARNING, -} - -//日志全局变量 -var G_Log_V *Log_T - -//全局once -var G_Once_V sync.Once - -//目前是否已经写入刷盘操作channel（保证全局只能写入一次，防止多协程操作阻塞） -var G_Flush_Log_Flag bool = false - -//控制 G_Flush_Log_Flag 的全局锁 -var G_Flush_Lock *sync.Mutex = &sync.Mutex{} - -/** -* 提供给协程调用的入口函数 -* -* @param RunConfigMap 是需要传递进来的配置信息key=>val的map数据 -*　调用示例： -* -//注意本调用必须在单独协程里运行 -m := map[string]string { - "log_notice_file_path": "log/koala.log" - "log_debug_file_path": "log/koala.log" - "log_trace_file_path": "log/koala.log" - "log_fatal_file_path": "log/koala.log.wf" - "log_warning_file_path": "log/koala.log.wf" - "log_cron_time": "day" - "log_chan_buff_size": "10240" - "log_flush_timer": "1" -} -go Log_Run(m) - -* 注意： -* 需要传递进来的配置是有要求的，必须是包含这些配置选项，否则会报错 -* -*/ -func Log_Run(RunConfigMap map[string]string) { - //初始化全局变量 - if G_Log_V == nil { - G_Log_V = new(Log_T) - } - - //设置配置map数据 - G_Log_V.RunConfigMap = RunConfigMap - - //调用初始化操作，全局只运行一次 - G_Once_V.Do(Log_Init) - - //启动log文件清理协程，定期删除过期的log文件 - go LogFile_CleanUP(int64(G_Log_V.LogLifeTime * 3600 * 24)) - - //永远循环等待channel的日志数据 - var log_msg Log_Msg_T - //var num int64 - for { - //监控是否有可以日志可以存取 - select { - case log_msg = <-G_Log_V.LogChan: - Log_Write_File(log_msg) - //if Log_Is_Debug() { - // Log_Debug_Print("G_Log_V.LogChan Length:", len(G_Log_V.LogChan)) - //} - default: - //breakLogChan长度 - //println("In Default ", num) - //打印目前G_Log_V的数据 - //if Log_Is_Debug() { - // Log_Debug_Print("G_Log_V.LogChan Length:", len(G_Log_V.LogChan)) - //} - time.Sleep(time.Duration(G_Log_V.LogFlushTimer) * time.Millisecond) - } - - //监控刷盘timer - //log_timer := time.NewTimer(time.Duration(G_Log_V.LogFlushTimer) * time.Millisecond) - select { - //超过设定时间开始检测刷盘（保证不会频繁写日志操作） - //case <-log_timer.C: - // log_timer.Stop() - // break - //如果收到刷盘channel的信号则刷盘且全局标志状态为 - case <-G_Log_V.FlushLogChan: - G_Flush_Lock.Lock() - G_Flush_Log_Flag = false - G_Flush_Lock.Unlock() - - //log_timer.Stop() - break - default: - break - } - - } -} - -/** - * 初始化Log协程相关操作 - * - * 注意： - * 全局操作, 只能协程初始化的时候调用一次 - * - */ -func Log_Init() { - if G_Log_V.RunConfigMap == nil { - errors.New("Log_Init fail: RunConfigMap data is nil") - } - - //构建日志文件名和文件句柄map内存 - G_Log_V.LogFilePath = make(map[int]string, len(G_Log_Type_Map)) - - //判断各个配置选项是否存在 - for conf_item_key, _ := range G_Conf_Item_Map { - if _, ok := G_Log_V.RunConfigMap[conf_item_key]; !ok { - errors.New(fmt.Sprintf("Log_Init fail: RunConfigMap not include item: %s", conf_item_key)) - } - } - - //扫描所有配置选项赋值给结构体 - var err error - var item_val_str string - var item_val_num int - for conf_item_k, conf_item_v := range G_Conf_Item_Map { - //对所有配置选项 进行类型转换 - if conf_item_v == LOG_CONF_TYPE_STR { - item_val_str = string(G_Log_V.RunConfigMap[conf_item_k]) - } else if conf_item_v == LOG_CONF_TYPE_NUM { - if item_val_num, err = strconv.Atoi(G_Log_V.RunConfigMap[conf_item_k]); err != nil { - errors.New(fmt.Sprintf("Log conf read map[%s] fail, map is error", conf_item_k)) - } - } - //进行各选项赋值 - switch conf_item_k { - //日志文件路径 - case LOG_CONF_NOTICE_FILE_PATH: - G_Log_V.LogFilePath[LOG_TYPE_NOTICE] = item_val_str - case LOG_CONF_DEBUG_FILE_PATH: - G_Log_V.LogFilePath[LOG_TYPE_DEBUG] = item_val_str - case LOG_CONF_TRACE_FILE_PATH: - G_Log_V.LogFilePath[LOG_TYPE_TRACE] = item_val_str - case LOG_CONF_FATAL_FILE_PATH: - G_Log_V.LogFilePath[LOG_TYPE_FATAL] = item_val_str - case LOG_CONF_WARNING_FILE_PATH: - G_Log_V.LogFilePath[LOG_TYPE_WARNING] = item_val_str - - //其他配置选项 - case LOG_CONF_CRON_TIME: - G_Log_V.LogCronTime = item_val_str - case LOG_CONF_CHAN_BUFFSIZE: - G_Log_V.LogChanBuffSize = item_val_num - case LOG_CONF_FLUSH_TIMER: - G_Log_V.LogFlushTimer = item_val_num - case LOG_CONF_DEBUG_OPEN: - if item_val_num == 1 { - G_Log_V.LogDebugOpen = true - } else { - G_Log_V.LogDebugOpen = false - } - case LOG_CONF_LEVEL: - G_Log_V.LogLevel = item_val_num - case LOG_CONF_FILE_LIFE_TIME: - G_Log_V.LogLifeTime = item_val_num - } - } - - //设置日志channel buffer - if G_Log_V.LogChanBuffSize <= 0 { - G_Log_V.LogChanBuffSize = 1024 - } - G_Log_V.LogChan = make(chan Log_Msg_T, G_Log_V.LogChanBuffSize) - - //初始化唯一的日志文件名和fd - G_Log_V.MergeLogFile = make(map[string]string, len(G_Log_Type_Map)) - G_Log_V.MergeLogFd = make(map[string]*os.File, len(G_Log_Type_Map)) - for _, log_file_path := range G_Log_V.LogFilePath { - G_Log_V.MergeLogFile[log_file_path] = "" - G_Log_V.MergeLogFd[log_file_path] = nil - } - - //打印目前G_Log_V的数据 - if Log_Is_Debug() { - Log_Debug_Print("G_Log_V data:", G_Log_V) - } - - // 设置清理时间不可为0 - if G_Log_V.LogLifeTime <= 0 { - G_Log_V.LogLifeTime = 7 // 默认7天 - } - -} - -/** - * 写日志操作 - * - */ -func Log_Write_File(log_msg Log_Msg_T) { - //读取多少行开始写日志 - //var max_line_num int - - //临时变量 - var ( - //动态生成需要最终输出的日志map - log_map map[string][]string - //读取单条的日志消息 - log_msg_var Log_Msg_T - //读取单个配置的日志文件名 - conf_file_name string - - write_buf string - line string - ) - - //打开文件 - Log_Open_File() - - //初始化map数据都为 - log_map = make(map[string][]string, len(G_Conf_FileToType_Map)) - for conf_file_name, _ = range G_Log_V.MergeLogFile { - log_map[conf_file_name] = []string{} - } - //fmt.Println(log_map) - - //压入第一条读取的日志(上游select读取的) - conf_file_name = G_Log_V.LogFilePath[log_msg.LogType] - log_map[conf_file_name] = []string{log_msg.LogData} - //fmt.Println(log_map) - - //读取日志(所有可读的日志都读取，然后按照需要打印的文件压入到不同map数组) - select { - case log_msg_var = <-G_Log_V.LogChan: - conf_file_name = G_Log_V.LogFilePath[log_msg_var.LogType] - log_map[conf_file_name] = append(log_map[conf_file_name], log_msg_var.LogData) - default: - break - } - //调试信息 - if Log_Is_Debug() { - Log_Debug_Print("Log Map:", log_map) - } - - //写入所有日志(所有map所有文件的都写) - for conf_file_name, _ = range G_Log_V.MergeLogFile { - if len(log_map[conf_file_name]) > 0 { - write_buf, line = "", "" - for _, line = range log_map[conf_file_name] { - write_buf += line - } - _, _ = G_Log_V.MergeLogFd[conf_file_name].WriteString(write_buf) - _ = G_Log_V.MergeLogFd[conf_file_name].Sync() - - //调试信息 - if Log_Is_Debug() { - Log_Debug_Print("Log String:", write_buf) - } - } - } - -} - -/** - * 打开&切割日志文件 - * - */ -func Log_Open_File() error { - var ( - file_suffix string - err error - conf_file_name string - run_file_name string - new_log_file_name string - new_log_file_fd *os.File - ) - - //构造日志文件名 - file_suffix = Log_Get_File_Suffix(time.Now()) - - //把重复日志文件都归一，然后进行相应日志文件的操作 - for conf_file_name, run_file_name = range G_Log_V.MergeLogFile { - new_log_file_name = fmt.Sprintf("%s.%s", conf_file_name, file_suffix) - - //如果新旧文件名不同，说明需要切割文件了(第一次运行则是全部初始化文件) - if new_log_file_name != run_file_name { - //关闭旧日志文件 - if G_Log_V.MergeLogFd[conf_file_name] != nil { - if err = G_Log_V.MergeLogFd[conf_file_name].Close(); err != nil { - errors.New(fmt.Sprintf("Close log file %s fail", run_file_name)) - } - } - //初始化新日志文件 - G_Log_V.MergeLogFile[conf_file_name] = new_log_file_name - G_Log_V.MergeLogFd[conf_file_name] = nil - - //创建&打开新日志文件 - new_log_file_fd, err = os.OpenFile(new_log_file_name, os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - errors.New(fmt.Sprintf("Open log file %s fail", new_log_file_name)) - } - new_log_file_fd.Seek(0, os.SEEK_END) - - //把处理的相应的结果进行赋值 - G_Log_V.MergeLogFile[conf_file_name] = new_log_file_name - G_Log_V.MergeLogFd[conf_file_name] = new_log_file_fd - } - } - - //调试 - //fmt.Println(G_Log_V) - - return nil -} - -/** - * 获取日志文件的切割时间 - * - * 说明： - * 目前主要支持三种粒度的设置，基本这些粒度足够我们使用了 - * 1天:day; 1小时:hour; 10分钟:ten - */ -func Log_Get_File_Suffix(now time.Time) string { - var file_suffix string - //now := time.Now() - - switch G_Log_V.LogCronTime { - - //按照天切割日志 - case "day": - file_suffix = now.Format("20060102") - - //按照小时切割日志 - case "hour": - file_suffix = now.Format("20060102_15") - - //按照10分钟切割日志 - case "ten": - file_suffix = fmt.Sprintf("%s%d0", now.Format("20060102_15"), int(now.Minute()/10)) - - //缺省按照小时 - default: - file_suffix = now.Format("20060102_15") - } - - return file_suffix -} - -/** - * 获取目前是否是Debug模式 - * - */ -func Log_Is_Debug() bool { - if G_Log_V.LogDebugOpen { - return true - } - return false -} - -/** - * 日志打印输出到终端函数 - * - */ -func Log_Debug_Print(msg string, v interface{}) { - - //获取调用的 函数/文件名/行号 等信息 - fcName, log_filename, log_lineno, ok := runtime.Caller(1) - if !ok { - errors.New("call runtime.Caller() fail") - } - log_callfunc := runtime.FuncForPC(fcName).Name() - - os_path_separator := Log_Get_Os_Separator(log_filename) - call_path := strings.Split(log_filename, os_path_separator) - if path_len := len(call_path); path_len > 2 { - log_filename = strings.Join(call_path[path_len-2:], os_path_separator) - } - - fmt.Println("\n=======================Log Debug Info Start=======================") - fmt.Println("[ call=", log_callfunc, "file=", log_filename, "no=", log_lineno, "]") - if msg != "" { - fmt.Println(msg) - } - fmt.Println(v) - fmt.Println("=======================Log Debug Info End=======================\n") -} - -/** - * 获取当前操作系统的路径切割符 - * - * 说明: 主要为了解决 os.PathSeparator有些时候无法满足要求的问题 - * - */ -func Log_Get_Os_Separator(path_name string) string { - //判断当前操作系统路径分割符 - var os_path_separator = "/" - if strings.ContainsAny(path_name, "\\") { - os_path_separator = "\\" - } - return os_path_separator -} - -/** - * 对notice日志，进行定期清理，执行周期等同于“日志切割周期” - */ -func LogFile_CleanUP(file_lifetime int64) { - - // println("clean up gorouting start!") - - // 5秒后再启动“清理”循环，错开启动初期的不稳定时段 - time.Sleep(time.Duration(5) * time.Second) - - var ( - // 清理周期，秒 - cycle_time int64 - // log文件保存周期，秒； 30天 - // file_lifetime int64 = 3600 * 24 * 30 - ) - - // 清理周期，设置为 “日志切割时间 ” - switch G_Log_V.LogCronTime { - case "day": - cycle_time = 3600 * 24 - case "hour": - cycle_time = 3600 - case "ten": - cycle_time = 600 - default: - cycle_time = 3600 - } - - // 目前，仅针对notice日志（所在log文件）进行删除操作 - conf_file_name := G_Log_V.LogFilePath[LOG_TYPE_NOTICE] - - // 删除log文件无限循环 - for { - var cleanUp_time time.Time = time.Unix(time.Now().Unix()-file_lifetime, 0) - - // 计算出，待删除log文件名 - file_suffix := Log_Get_File_Suffix(cleanUp_time) - log_file_name := fmt.Sprintf("%s.%s", conf_file_name, file_suffix) - - // println(log_file_name) - - // 删除log文件 - // if err := os.Remove(log_file_name); err != nil { - // println(err.Error()) - // } - - os.Remove(log_file_name) - // 等待下一个清理周期，sleep - time.Sleep(time.Duration(cycle_time) * time.Second) - } -} diff --git a/src/utility/network/http.go b/src/utility/network/http.go deleted file mode 100644 index e31337a..0000000 --- a/src/utility/network/http.go +++ /dev/null @@ -1,671 +0,0 @@ -/** - * Koala Rule Engine Core - * - * @package: main - * @desc: koala engine http server - * - * @author: heiyeluren - * @github: https://github.com/heiyeluren - * @blog: https://blog.csdn.net/heiyeshuwu - * - */ - -package network - -import ( - "bytes" - "errors" - "net/url" - "strconv" - "strings" - "time" -) - -type HttpClient struct { - debug bool - httpConnection *HttpConnection -} - -func NewHttpClient() *HttpClient { - return &HttpClient{} -} - -func (this *HttpClient) SetDebug(debug bool) { - this.debug = debug -} - -func (this *HttpClient) Dial(localAddress string, remoteAddress string, timeout time.Duration) error { - tcpConnection, err := TcpConnect(localAddress, remoteAddress, timeout) - if err != nil { - return err - } - this.httpConnection = NewHttpConnection(tcpConnection) - return nil -} - -func (this *HttpClient) ConnectTo(address string, timeout time.Duration) error { - tcpConnection, err := TcpConnect("", address, timeout) - if err != nil { - return err - } - this.httpConnection = NewHttpConnection(tcpConnection) - return nil -} - -func (this *HttpClient) Close() error { - return this.httpConnection.Close() -} - -func (this *HttpClient) Send(request *HttpRequest, writeTimeout time.Duration) error { - return this.httpConnection.WriteRequest(request, writeTimeout) -} - -func (this *HttpClient) Recv(readTimeout time.Duration) (*HttpResponse, error) { - return this.httpConnection.ReadResponse(readTimeout) -} - -func (this *HttpClient) Get(uri string, readTimeout time.Duration, writeTimeout time.Duration) (*HttpResponse, error) { - request := NewHttpRequest() - request.SetMethod("GET") - request.SetUri(uri) - request.SetVersion("HTTP/1.1") - if this.debug { - println(request.String()) - } - if err := this.Send(request, writeTimeout); err != nil { - return nil, err - } - return this.Recv(readTimeout) -} - -func (this *HttpClient) Post(uri string, posts map[string]string, readTimeout time.Duration, writeTimeout time.Duration) (*HttpResponse, error) { - request := NewHttpRequest() - request.SetMethod("POST") - request.SetUri(uri) - request.SetVersion("HTTP/1.1") - request.SetPosts(posts) - if this.debug { - println(request.String()) - } - if err := this.Send(request, writeTimeout); err != nil { - return nil, err - } - return this.Recv(readTimeout) -} - -const ( - httpHeadBodySeparator = "\r\n\r\n" - httpHeadBodySeparatorLen = len(httpHeadBodySeparator) - httpHeaderPartSeparator = " " - httpHeaderPropSeparator = ":" - httpHeaderSeparator = "\r\n" -) - -type HttpConnection struct { - *TcpConnection // 内嵌 *TcpConnection - requestStream []byte // 保存未读的请求数据流 - responseStream []byte // 保存未读的响应数据流 - headBuffer []byte // 解析请求/响应的头部时使用的临时缓冲区 - bodyBuffer []byte // 解析请求/响应的内容时使用的临时缓冲区 -} - -const ( - headBufferSize = 512 - bodyBufferSize = 1024 -) - -func NewHttpConnection(tcpConnection *TcpConnection) *HttpConnection { - return &HttpConnection{TcpConnection: tcpConnection, headBuffer: make([]byte, headBufferSize), bodyBuffer: make([]byte, bodyBufferSize)} -} - -func (this *HttpConnection) ReadRequest(timeout time.Duration) (*HttpRequest, error) { - if err := this.SetReadDeadline(time.Now().Add(timeout)); err != nil { - return nil, err - } - request, err := this.readMessageTo(NewHttpRequest(), &this.requestStream) - if request == nil { - return nil, err - } - // 获取 remote addr，客户端ip地址 - request.(*HttpRequest).remoteAddr = this.TCPConn.RemoteAddr().String() - return request.(*HttpRequest), err -} - -func (this *HttpConnection) ReadResponse(timeout time.Duration) (*HttpResponse, error) { - if err := this.SetReadDeadline(time.Now().Add(timeout)); err != nil { - return nil, err - } - response, err := this.readMessageTo(NewHttpResponse(), &this.responseStream) - if response == nil { - return nil, err - } - return response.(*HttpResponse), err -} - -func (this *HttpConnection) WriteRequest(request *HttpRequest, timeout time.Duration) error { - if err := this.SetWriteDeadline(time.Now().Add(timeout)); err != nil { - return err - } - return this.writeStream(request.Stream()) -} - -func (this *HttpConnection) WriteResponse(response *HttpResponse, timeout time.Duration) error { - if err := this.SetWriteDeadline(time.Now().Add(timeout)); err != nil { - return err - } - return this.writeStream(response.Stream()) -} - -func (this *HttpConnection) readMessageTo(httpMessage HttpMessage, stream *[]byte) (HttpMessage, error) { - for *stream == nil || bytes.Index(*stream, []byte(httpHeadBodySeparator)) == -1 { - if err := this.readToStream(stream, this.headBuffer); err != nil { - return nil, err - } - } - position := bytes.Index(*stream, []byte(httpHeadBodySeparator)) - headStream := (*stream)[0:position] - headLen := len(headStream) - if len(*stream) > headLen+httpHeadBodySeparatorLen { - *stream = (*stream)[position+httpHeadBodySeparatorLen:] - } else { - *stream = nil - } - if err := parseHeadStreamTo(httpMessage, headStream); err != nil { - return nil, err - } - contentLength, err := strconv.Atoi(httpMessage.Header("Content-Length")) - if err != nil { - contentLength = 0 - } - if contentLength > 1<<20 { - return nil, errors.New("body too large!") - } - if contentLength > 0 { - for *stream == nil || len(*stream) < contentLength { - if err := this.readToStream(stream, this.bodyBuffer); err != nil { - return nil, err - } - } - bodyStream := (*stream)[:contentLength] - if err := httpMessage.parseBodyStream(bodyStream); err != nil { - return nil, err - } - if len(*stream) == contentLength { - *stream = nil - } else { - *stream = (*stream)[contentLength:] - } - } - return httpMessage, nil -} - -func (this *HttpConnection) writeStream(stream []byte) error { - _, err := this.TcpConnection.Write(stream) - return err -} - -func (this *HttpConnection) readToStream(stream *[]byte, buffer []byte) error { - n, err := this.TcpConnection.Read(buffer) - if n > 0 { - *stream = append(*stream, buffer[:n]...) - } - if err != nil { - return err - } - return nil -} - -type HttpMessage interface { - SetHeader(key string, value string) - Header(key string) string - Stream() []byte - parseHairStream(hairStream []byte) error - parseBodyStream(bodyStream []byte) error -} - -func parseHeadStreamTo(message HttpMessage, headStream []byte) error { - lines := bytes.Split(headStream, []byte(httpHeaderSeparator)) - lineCount := len(lines) - if lineCount == 0 { - return errors.New("Bad http message") - } - message.parseHairStream(lines[0]) - if lineCount > 1 { - lines = lines[1:] - for _, line := range lines { - parts := bytes.SplitN(line, []byte(httpHeaderPropSeparator), 2) - if len(parts) != 2 { - return errors.New("Bad http message") - } - message.SetHeader(strings.Trim(string(parts[0]), " "), strings.Trim(string(parts[1]), " ")) - } - } - return nil -} - -type HttpRequest struct { - remoteAddr string - method string - uri string - version string - headers map[string]string - pathInfo string - gets map[string]string - posts map[string]string - cookies map[string]string - vars map[string]interface{} -} - -func NewHttpRequest() *HttpRequest { - return &HttpRequest{ - method: "GET", - uri: "/", - version: "HTTP/1.1", - pathInfo: "/", - headers: make(map[string]string), - gets: make(map[string]string), - posts: make(map[string]string), - cookies: make(map[string]string), - vars: make(map[string]interface{}), - } -} - -func (this *HttpRequest) GetRemoteIP() string { - parts := strings.SplitN(this.remoteAddr, ":", 2) - return parts[0] -} - -func (this *HttpRequest) Method() string { - return this.method -} - -func (this *HttpRequest) SetMethod(method string) { - this.method = method -} - -func (this *HttpRequest) Uri() string { - return this.uri -} - -func (this *HttpRequest) SetUri(uri string) { - this.uri = uri - if markPos := strings.Index(this.uri, "?"); markPos != -1 { - this.pathInfo = this.uri[:markPos] - } -} - -func (this *HttpRequest) PathInfo() string { - return this.pathInfo -} - -func (this *HttpRequest) Version() string { - return this.version -} - -func (this *HttpRequest) SetVersion(version string) { - this.version = version -} - -func (this *HttpRequest) IsGet() bool { - return this.Method() == "GET" -} - -func (this *HttpRequest) IsPost() bool { - return this.Method() == "POST" -} - -func (this *HttpRequest) IsPut() bool { - return this.Method() == "PUT" -} - -func (this *HttpRequest) IsDelete() bool { - return this.Method() == "DELETE" -} - -func (this *HttpRequest) Gets() map[string]string { - return this.gets -} - -func (this *HttpRequest) Ghas(key string) bool { - if _, ok := this.gets[key]; ok { - return true - } - return false -} - -func (this *HttpRequest) Gstr(key string) string { - if value, ok := this.gets[key]; ok { - return value - } - return "" -} - -func (this *HttpRequest) Gint(key string) int { - value := this.Gstr(key) - if value == "" { - return 0 - } - i, err := strconv.Atoi(value) - if err != nil { - return 0 - } - return i -} - -func (this *HttpRequest) Posts() map[string]string { - return this.posts -} - -func (this *HttpRequest) Phas(key string) bool { - if _, ok := this.posts[key]; ok { - return true - } - return false -} - -func (this *HttpRequest) Pstr(key string) string { - if value, ok := this.posts[key]; ok { - return value - } - return "" -} - -func (this *HttpRequest) Pint(key string) int { - value := this.Pstr(key) - if value == "" { - return 0 - } - i, err := strconv.Atoi(value) - if err != nil { - return 0 - } - return i -} - -func (this *HttpRequest) Rint(key string) int { - if value := this.Pint(key); value != 0 { - return value - } - return this.Gint(key) -} - -func (this *HttpRequest) Rstr(key string) string { - if value := this.Pstr(key); value != "" { - return value - } - return this.Gstr(key) -} - -func (this *HttpRequest) SetPosts(posts map[string]string) { - this.posts = posts -} - -func (this *HttpRequest) Cookies() map[string]string { - return this.cookies -} - -func (this *HttpRequest) Chas(key string) bool { - if _, ok := this.cookies[key]; ok { - return true - } - return false -} - -func (this *HttpRequest) Cstr(key string) string { - if value, ok := this.cookies[key]; ok { - return value - } - return "" -} - -func (this *HttpRequest) Cint(key string) int { - value := this.Cstr(key) - if value == "" { - return 0 - } - i, err := strconv.Atoi(value) - if err != nil { - return 0 - } - return i -} - -func (this *HttpRequest) IsKeepAlive() bool { - return this.version == "HTTP/1.1" && this.Header("Connection") != "close" -} - -func (this *HttpRequest) Header(key string) string { - if value, ok := this.headers[key]; ok { - return value - } - return "" -} - -func (this *HttpRequest) SetHeader(key string, value string) { - this.headers[key] = value -} - -func (this *HttpRequest) Var(key string) interface{} { - return this.vars[key] -} - -func (this *HttpRequest) SetVar(key string, value interface{}) { - this.vars[key] = value -} - -func (this *HttpRequest) Stream() []byte { - body := "" - if this.method == "POST" { - postsLen := len(this.posts) - if postsLen > 0 { - pairs := make([]string, 0, postsLen) - for key, value := range this.posts { - pairs = append(pairs, url.QueryEscape(key)+"="+url.QueryEscape(value)) - } - body = strings.Join(pairs, "&") - } - this.headers["Content-Length"] = strconv.Itoa(len(body)) - } - // @todo: 需要优化 - s := this.method + httpHeaderPartSeparator + this.uri + httpHeaderPartSeparator + this.version + httpHeaderSeparator - for key, value := range this.headers { - s += key + httpHeaderPropSeparator + " " + value + httpHeaderSeparator - } - s += httpHeaderSeparator + body - return []byte(s) -} - -func (this *HttpRequest) String() string { - return string(this.Stream()) -} - -func (this *HttpRequest) parseHairStream(hairStream []byte) error { - parts := bytes.SplitN(hairStream, []byte(httpHeaderPartSeparator), 3) - if len(parts) != 3 { - return errors.New("Bad http request") - } - this.method, this.uri, this.version = string(parts[0]), string(parts[1]), string(parts[2]) - this.pathInfo = this.uri - if markPos := strings.Index(this.uri, "?"); markPos != -1 { - this.pathInfo = this.uri[0:markPos] - if markPos != len(this.uri)-1 { - queryString := this.uri[markPos+1:] - kvStrings := strings.Split(queryString, "&") - for _, kvString := range kvStrings { - parts := strings.SplitN(kvString, "=", 2) - if len(parts) != 2 { - return errors.New("Bad http request") - } - key, err := url.QueryUnescape(parts[0]) - if err != nil { - key = parts[0] - } - value, err := url.QueryUnescape(parts[1]) - if err != nil { - value = parts[1] - } - this.gets[key] = value - } - } - } - return nil -} - -func (this *HttpRequest) parseBodyStream(bodyStream []byte) error { - kvStrings := strings.Split(string(bodyStream), "&") - for _, kvString := range kvStrings { - parts := strings.SplitN(kvString, "=", 2) - if len(parts) != 2 { - return errors.New("Bad http request") - } - key, err := url.QueryUnescape(parts[0]) - if err != nil { - key = parts[0] - } - value, err := url.QueryUnescape(parts[1]) - if err != nil { - value = parts[1] - } - this.posts[key] = value - } - return nil -} - -type HttpResponse struct { - version string - code int - phrase string - headers map[string]string - bodyStream []byte -} - -var httpStatus = map[int]string{ - 200: "OK", - 301: "Moved Permanently", - 302: "Moved Temporarily", - 400: "Bad Request", - 401: "Unauthorized", - 403: "Forbidden", - 404: "Not Found", - 500: "Internal Server Error", -} - -func NewHttpResponse() *HttpResponse { - return &HttpResponse{version: "HTTP/1.1", code: 200, phrase: httpStatus[200], headers: map[string]string{ - "Server": "Koala-Server", - "Content-Type": "text/html; charset=utf-8", - "Connection": "keep-alive", - }} -} - -func (this *HttpResponse) Version() string { - return this.version -} - -func (this *HttpResponse) SetVersion(version string) { - this.version = version -} - -func (this *HttpResponse) Code() int { - return this.code -} - -func (this *HttpResponse) SetCode(code int) { - this.code = code - this.phrase = httpStatus[this.code] -} - -func (this *HttpResponse) Status() (int, string) { - return this.code, this.phrase -} - -func (this *HttpResponse) SetStatus(code int, phrase string) { - this.code, this.phrase = code, phrase -} - -func (this *HttpResponse) Header(key string) string { - if value, ok := this.headers[key]; ok { - return value - } - return "" -} - -func (this *HttpResponse) SetHeader(key string, value string) { - this.headers[key] = value -} - -func (this *HttpResponse) IsConnectionClose() bool { - return this.Version() != "HTTP/1.1" || this.Header("Connection") == "close" -} - -func (this *HttpResponse) SetBodyStream(bodyStream []byte) { - this.bodyStream = bodyStream -} - -func (this *HttpResponse) BodyStream() []byte { - return this.bodyStream -} - -func (this *HttpResponse) SetBodyString(bodyString string) { - this.SetBodyStream([]byte(bodyString)) -} - -func (this *HttpResponse) BodyString() string { - return string(this.bodyStream) -} - -func (this *HttpResponse) Putb(stream []byte) { - if len(stream) > 0 { - this.bodyStream = append(this.bodyStream, stream...) - } -} - -func (this *HttpResponse) Puts(content string) { - this.bodyStream = append(this.bodyStream, content...) -} - -func (this *HttpResponse) Stream() []byte { - var b bytes.Buffer - b.WriteString(this.version) - b.WriteString(httpHeaderPartSeparator) - b.WriteString(strconv.Itoa(this.code)) - b.WriteString(httpHeaderPartSeparator) - b.WriteString(this.phrase) - b.WriteString(httpHeaderSeparator) - this.headers["Content-Length"] = strconv.Itoa(len(this.bodyStream)) - for key, value := range this.headers { - b.WriteString(key) - b.WriteString(httpHeaderPropSeparator + " ") - b.WriteString(value) - b.WriteString(httpHeaderSeparator) - } - b.WriteString(httpHeaderSeparator) - if len(this.bodyStream) > 0 { - b.Write(this.bodyStream) - } - return b.Bytes() -} - -func (this *HttpResponse) String() string { - return string(this.Stream()) -} - -func (this *HttpResponse) parseHairStream(hairStream []byte) error { - parts := bytes.SplitN(hairStream, []byte(httpHeaderPartSeparator), 3) - if len(parts) != 3 { - return errors.New("Bad response") - } - this.version = string(parts[0]) - if code, err := strconv.Atoi(string(parts[1])); err == nil { - this.code = code - } else { - this.code = 500 - } - this.phrase = string(parts[2]) - return nil -} - -func (this *HttpResponse) parseBodyStream(bodyStream []byte) error { - this.bodyStream = bodyStream - return nil -} diff --git a/src/utility/network/tcp.go b/src/utility/network/tcp.go deleted file mode 100644 index c74359d..0000000 --- a/src/utility/network/tcp.go +++ /dev/null @@ -1,209 +0,0 @@ -/** - * Koala Rule Engine Core - * - * @package: main - * @desc: koala engine tcp server - * - * @author: heiyeluren - * @github: https://github.com/heiyeluren - * @blog: https://blog.csdn.net/heiyeshuwu - * - */ - -package network - -import ( - "encoding/gob" - "errors" - "net" - "time" -) - -// 监听一个 tcp 地址 -func TcpListen(address string) (*TcpListener, error) { - tcpAddr, err := net.ResolveTCPAddr("tcp", address) - if err != nil { - return nil, err - } - tcpListener, err := net.ListenTCP("tcp", tcpAddr) - if err != nil { - return nil, err - } - return newTcpListener(tcpListener), nil -} - -// tcp 监听器 -type TcpListener struct { - *net.TCPListener -} - -func newTcpListener(tcpListener *net.TCPListener) *TcpListener { - return &TcpListener{TCPListener: tcpListener} -} - -func (this *TcpListener) Accept() (*TcpConnection, error) { - tcpConn, err := this.TCPListener.AcceptTCP() - if err != nil { - return nil, err - } - return newTcpConnection(tcpConn), nil -} - -// 连接一个 tcp 地址 -func TcpConnect(localAddress string, remoteAddress string, timeout time.Duration) (*TcpConnection, error) { - // @todo: 本 api 调用不绑定 localAddress，可能会影响上层应用程序 - conn, err := net.DialTimeout("tcp", remoteAddress, timeout) - if err != nil { - return nil, err - } - tcpConn := conn.(*net.TCPConn) - return newTcpConnection(tcpConn), nil -} - -// tcp 连接 -type TcpConnection struct { - *net.TCPConn - protoBuffer []byte -} - -// 创建 tcp 连接对象 -func newTcpConnection(tcpConn *net.TCPConn) *TcpConnection { - return &TcpConnection{TCPConn: tcpConn} -} - -func (this *TcpConnection) ReadStream(stream []byte, count int) error { - if len(stream) < count { - return errors.New("bad stream") - } - stream = stream[:count] - left := count - for left > 0 { - n, err := this.Read(stream) - if n > 0 { - left -= n - if left > 0 { - stream = stream[n:] - } - } - if err != nil { - return err - } - } - return nil -} - -const protoBufferLen = 8 - -func (this *TcpConnection) ReadProtoBuffer() error { - this.protoBuffer = make([]byte, protoBufferLen) - left := protoBufferLen - for left > 0 { - n, err := this.TCPConn.Read(this.protoBuffer) - if n > 0 { - left -= n - } - if err != nil { - return err - } - } - return nil -} - -func (this *TcpConnection) IsHttpProto() bool { - if this.protoBuffer == nil { - return false - } - if this.protoBuffer[0] == 'G' && this.protoBuffer[1] == 'E' && this.protoBuffer[2] == 'T' && this.protoBuffer[3] == ' ' && this.protoBuffer[4] == '/' { - return true - } - if this.protoBuffer[0] == 'P' && this.protoBuffer[1] == 'O' && this.protoBuffer[2] == 'S' && this.protoBuffer[3] == 'T' && this.protoBuffer[4] == ' ' && this.protoBuffer[5] == '/' { - return true - } - return false -} - -func (this *TcpConnection) Read(p []byte) (n int, err error) { - if this.protoBuffer == nil { - n, err = this.TCPConn.Read(p) - } else { - pLen := len(p) - if pLen < protoBufferLen { - copy(p, this.protoBuffer[0:pLen]) - this.protoBuffer = this.protoBuffer[pLen:] - n = pLen - } else { - copy(p, this.protoBuffer) - this.protoBuffer = nil - n = protoBufferLen - } - } - return -} - -func (this *TcpConnection) ReadRpcRequest(timeout time.Duration) (*RpcRequest, error) { - if err := this.SetReadDeadline(time.Now().Add(timeout)); err != nil { - return nil, err - } - decoder := gob.NewDecoder(this) - request := NewRpcRequest() - if err := decoder.Decode(request); err != nil { - return nil, err - } - return request, nil -} - -func (this *TcpConnection) WriteRpcResponse(response *RpcResponse, timeout time.Duration) error { - if err := this.SetWriteDeadline(time.Now().Add(timeout)); err != nil { - return err - } - encoder := gob.NewEncoder(this) - if err := encoder.Encode(response); err != nil { - return err - } - return nil -} - -func (this *TcpConnection) Rpc(request *RpcRequest, readTimeout time.Duration, writeTimeout time.Duration) (*RpcResponse, error) { - if err := this.SetWriteDeadline(time.Now().Add(writeTimeout)); err != nil { - return nil, err - } - encoder := gob.NewEncoder(this) - if err := encoder.Encode(request); err != nil { - return nil, err - } - if err := this.SetReadDeadline(time.Now().Add(readTimeout)); err != nil { - return nil, err - } - decoder := gob.NewDecoder(this) - response := NewRpcResponse() - if err := decoder.Decode(response); err != nil { - return nil, err - } - return response, nil -} - -// 注册 rpc 类型 -func RegisterRpcTypeForValue(value interface{}) { - gob.Register(value) -} - -type RpcRequest struct { - Func string - Args map[string]interface{} -} - -func NewRpcRequest() *RpcRequest { - return &RpcRequest{Args: make(map[string]interface{})} -} - -type RpcResponse struct { - Result bool - Reason string - Code int - Args map[string]interface{} - Data interface{} -} - -func NewRpcResponse() *RpcResponse { - return &RpcResponse{Args: make(map[string]interface{})} -} diff --git a/utility/configs.go b/utility/configs.go new file mode 100644 index 0000000..b306149 --- /dev/null +++ b/utility/configs.go @@ -0,0 +1,140 @@ +/** + * Koala Rule Engine Core + * + * @package: main + * @desc: koala engine - config file parse engine + * + * @author: heiyeluren + * @github: https://github.com/heiyeluren + * @blog: https://blog.csdn.net/heiyeshuwu + * + */ + +package utility + +import ( + "errors" + "io/ioutil" + "path" + "strconv" + "strings" +) + +// Config . +type Config struct { + data map[string]string +} + +// NewConfig . +func NewConfig() *Config { + return &Config{data: make(map[string]string)} +} + +const emptyRunes = " \r\t\v" + +// Load . +func (c *Config) Load(configFile string) error { + stream, err := ioutil.ReadFile(configFile) + if err != nil { + return errors.New("cannot load config file") + } + content := string(stream) + lines := strings.Split(content, "\n") + for _, line := range lines { + line = strings.Trim(line, emptyRunes) + // 去除空行和注释 + if line == "" || line[0] == '#' { + continue + } + parts := strings.SplitN(line, "=", 2) + if len(parts) == 2 { + for i, part := range parts { + parts[i] = strings.Trim(part, emptyRunes) + } + c.data[parts[0]] = parts[1] + } else { + // 判断并处理include条目，load相应的config文件 + // include 的配置文件应该在当前配置文件所在的目录下 + includes := strings.SplitN(parts[0], " ", 2) + if len(includes) == 2 && strings.EqualFold(includes[0], "include") { + // 拼解新包含config文件的path + confDir := path.Dir(configFile) + newConfName := strings.Trim(includes[1], emptyRunes) + newConfPath := path.Join(confDir, newConfName) + // 载入include的config文件，调用Load自身 + if err := c.Load(newConfPath); err != nil { + return errors.New("load include config file failed") + } + continue + } else { + return errors.New("invalid config file syntax") + } + } + } + return nil +} + +// GetAll . +func (c *Config) GetAll() map[string]string { + return c.data +} + +// Get . +func (c *Config) Get(key string) string { + if value, ok := c.data[key]; ok { + return value + } + return "" +} + +// GetInt . +func (c *Config) GetInt(key string) int { + value := c.Get(key) + if value == "" { + return 0 + } + result, err := strconv.Atoi(value) + if err != nil { + return 0 + } + return result +} + +// GetInt64 . +func (c *Config) GetInt64(key string) int64 { + value := c.Get(key) + if value == "" { + return 0 + } + result, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return 0 + } + return result +} + +// GetSlice . +func (c *Config) GetSlice(key string, separator string) []string { + var slice []string + value := c.Get(key) + if value != "" { + for _, part := range strings.Split(value, separator) { + slice = append(slice, strings.Trim(part, emptyRunes)) + } + } + return slice +} + +// GetSliceInt . +func (c *Config) GetSliceInt(key string, separator string) []int { + slice := c.GetSlice(key, separator) + var results []int + for _, part := range slice { + result, err := strconv.Atoi(part) + if err != nil { + continue + } + results = append(results, result) + } + return results +} diff --git a/utility/http.go b/utility/http.go new file mode 100644 index 0000000..c3a4a8e --- /dev/null +++ b/utility/http.go @@ -0,0 +1,744 @@ +/** + * Koala Rule Engine Core + * + * @package: main + * @desc: koala engine http server + * + * @author: heiyeluren + * @github: https://github.com/heiyeluren + * @blog: https://blog.csdn.net/heiyeshuwu + * + */ + +package utility + +import ( + "bytes" + "errors" + "net/url" + "strconv" + "strings" + "time" +) + +// HttpClient . +type HttpClient struct { + debug bool + httpConnection *HttpConnection +} + +// NewHttpClient . +func NewHttpClient() *HttpClient { + return &HttpClient{} +} + +// SetDebug . +func (c *HttpClient) SetDebug(debug bool) { + c.debug = debug +} + +// Dial . +func (c *HttpClient) Dial(localAddress string, remoteAddress string, timeout time.Duration) error { + tcpConnection, err := TcpConnect(localAddress, remoteAddress, timeout) + if err != nil { + return err + } + c.httpConnection = NewHttpConnection(tcpConnection) + return nil +} + +// ConnectTo . +func (c *HttpClient) ConnectTo(address string, timeout time.Duration) error { + tcpConnection, err := TcpConnect("", address, timeout) + if err != nil { + return err + } + c.httpConnection = NewHttpConnection(tcpConnection) + return nil +} + +// Close . +func (c *HttpClient) Close() error { + return c.httpConnection.Close() +} + +// Send . +func (c *HttpClient) Send(request *HttpRequest, writeTimeout time.Duration) error { + return c.httpConnection.WriteRequest(request, writeTimeout) +} + +// Recv . +func (c *HttpClient) Recv(readTimeout time.Duration) (*HttpResponse, error) { + return c.httpConnection.ReadResponse(readTimeout) +} + +// Get . +func (c *HttpClient) Get(uri string, readTimeout time.Duration, writeTimeout time.Duration) (*HttpResponse, error) { + request := NewHttpRequest() + request.SetMethod("GET") + request.SetUri(uri) + request.SetVersion("HTTP/1.1") + if c.debug { + println(request.String()) + } + if err := c.Send(request, writeTimeout); err != nil { + return nil, err + } + return c.Recv(readTimeout) +} + +// Post . +func (c *HttpClient) Post(uri string, posts map[string]string, readTimeout time.Duration, writeTimeout time.Duration) (*HttpResponse, error) { + request := NewHttpRequest() + request.SetMethod("POST") + request.SetUri(uri) + request.SetVersion("HTTP/1.1") + request.SetPosts(posts) + if c.debug { + println(request.String()) + } + if err := c.Send(request, writeTimeout); err != nil { + return nil, err + } + return c.Recv(readTimeout) +} + +const ( + httpHeadBodySeparator = "\r\n\r\n" + httpHeadBodySeparatorLen = len(httpHeadBodySeparator) + httpHeaderPartSeparator = " " + httpHeaderPropSeparator = ":" + httpHeaderSeparator = "\r\n" +) + +// HttpConnection . +type HttpConnection struct { + *TcpConnection // 内嵌 *TcpConnection + requestStream []byte // 保存未读的请求数据流 + responseStream []byte // 保存未读的响应数据流 + headBuffer []byte // 解析请求/响应的头部时使用的临时缓冲区 + bodyBuffer []byte // 解析请求/响应的内容时使用的临时缓冲区 +} + +const ( + headBufferSize = 512 + bodyBufferSize = 1024 +) + +// NewHttpConnection . +func NewHttpConnection(tcpConnection *TcpConnection) *HttpConnection { + return &HttpConnection{TcpConnection: tcpConnection, headBuffer: make([]byte, headBufferSize), bodyBuffer: make([]byte, bodyBufferSize)} +} + +// ReadRequest . +func (c *HttpConnection) ReadRequest(timeout time.Duration) (*HttpRequest, error) { + if err := c.SetReadDeadline(time.Now().Add(timeout)); err != nil { + return nil, err + } + request, err := c.readMessageTo(NewHttpRequest(), &c.requestStream) + if request == nil { + return nil, err + } + // 获取 remote addr，客户端ip地址 + request.(*HttpRequest).remoteAddr = c.TCPConn.RemoteAddr().String() + return request.(*HttpRequest), err +} + +// ReadResponse . +func (c *HttpConnection) ReadResponse(timeout time.Duration) (*HttpResponse, error) { + if err := c.SetReadDeadline(time.Now().Add(timeout)); err != nil { + return nil, err + } + response, err := c.readMessageTo(NewHttpResponse(), &c.responseStream) + if response == nil { + return nil, err + } + return response.(*HttpResponse), err +} + +// WriteRequest . +func (c *HttpConnection) WriteRequest(request *HttpRequest, timeout time.Duration) error { + if err := c.SetWriteDeadline(time.Now().Add(timeout)); err != nil { + return err + } + return c.writeStream(request.Stream()) +} + +// WriteResponse . +func (c *HttpConnection) WriteResponse(response *HttpResponse, timeout time.Duration) error { + if err := c.SetWriteDeadline(time.Now().Add(timeout)); err != nil { + return err + } + return c.writeStream(response.Stream()) +} + +// readMessageTo . +func (c *HttpConnection) readMessageTo(httpMessage HttpMessage, stream *[]byte) (HttpMessage, error) { + for *stream == nil || bytes.Index(*stream, []byte(httpHeadBodySeparator)) == -1 { + if err := c.readToStream(stream, c.headBuffer); err != nil { + return nil, err + } + } + position := bytes.Index(*stream, []byte(httpHeadBodySeparator)) + headStream := (*stream)[0:position] + headLen := len(headStream) + if len(*stream) > headLen+httpHeadBodySeparatorLen { + *stream = (*stream)[position+httpHeadBodySeparatorLen:] + } else { + *stream = nil + } + if err := parseHeadStreamTo(httpMessage, headStream); err != nil { + return nil, err + } + contentLength, err := strconv.Atoi(httpMessage.Header("Content-Length")) + if err != nil { + contentLength = 0 + } + if contentLength > 1<<20 { + return nil, errors.New("body too large") + } + if contentLength > 0 { + for *stream == nil || len(*stream) < contentLength { + if err := c.readToStream(stream, c.bodyBuffer); err != nil { + return nil, err + } + } + bodyStream := (*stream)[:contentLength] + if err := httpMessage.parseBodyStream(bodyStream); err != nil { + return nil, err + } + if len(*stream) == contentLength { + *stream = nil + } else { + *stream = (*stream)[contentLength:] + } + } + return httpMessage, nil +} + +func (c *HttpConnection) writeStream(stream []byte) error { + _, err := c.TcpConnection.Write(stream) + return err +} + +func (c *HttpConnection) readToStream(stream *[]byte, buffer []byte) error { + n, err := c.TcpConnection.Read(buffer) + if n > 0 { + *stream = append(*stream, buffer[:n]...) + } + if err != nil { + return err + } + return nil +} + +// HttpMessage . +type HttpMessage interface { + SetHeader(key string, value string) + Header(key string) string + Stream() []byte + parseHairStream(hairStream []byte) error + parseBodyStream(bodyStream []byte) error +} + +func parseHeadStreamTo(message HttpMessage, headStream []byte) error { + lines := bytes.Split(headStream, []byte(httpHeaderSeparator)) + lineCount := len(lines) + if lineCount == 0 { + return errors.New("bad http message") + } + message.parseHairStream(lines[0]) + if lineCount > 1 { + lines = lines[1:] + for _, line := range lines { + parts := bytes.SplitN(line, []byte(httpHeaderPropSeparator), 2) + if len(parts) != 2 { + return errors.New("bad http message") + } + message.SetHeader(strings.Trim(string(parts[0]), " "), strings.Trim(string(parts[1]), " ")) + } + } + return nil +} + +// HttpRequest . +type HttpRequest struct { + remoteAddr string + method string + uri string + version string + headers map[string]string + pathInfo string + gets map[string]string + posts map[string]string + cookies map[string]string + vars map[string]interface{} +} + +// NewHttpRequest . +func NewHttpRequest() *HttpRequest { + return &HttpRequest{ + method: "GET", + uri: "/", + version: "HTTP/1.1", + pathInfo: "/", + headers: make(map[string]string), + gets: make(map[string]string), + posts: make(map[string]string), + cookies: make(map[string]string), + vars: make(map[string]interface{}), + } +} + +// GetRemoteIP . +func (r *HttpRequest) GetRemoteIP() string { + parts := strings.SplitN(r.remoteAddr, ":", 2) + return parts[0] +} + +// Method . +func (r *HttpRequest) Method() string { + return r.method +} + +// SetMethod . +func (r *HttpRequest) SetMethod(method string) { + r.method = method +} + +// Uri . +func (r *HttpRequest) Uri() string { + return r.uri +} + +// SetUri . +func (r *HttpRequest) SetUri(uri string) { + r.uri = uri + if markPos := strings.Index(r.uri, "?"); markPos != -1 { + r.pathInfo = r.uri[:markPos] + } +} + +// PathInfo . +func (r *HttpRequest) PathInfo() string { + return r.pathInfo +} + +// Version . +func (r *HttpRequest) Version() string { + return r.version +} + +// SetVersion . +func (r *HttpRequest) SetVersion(version string) { + r.version = version +} + +// IsGet . +func (r *HttpRequest) IsGet() bool { + return r.Method() == "GET" +} + +// IsPost . +func (r *HttpRequest) IsPost() bool { + return r.Method() == "POST" +} + +// IsPut . +func (r *HttpRequest) IsPut() bool { + return r.Method() == "PUT" +} + +// IsDelete . +func (r *HttpRequest) IsDelete() bool { + return r.Method() == "DELETE" +} + +// Gets . +func (r *HttpRequest) Gets() map[string]string { + return r.gets +} + +// Ghas . +func (r *HttpRequest) Ghas(key string) bool { + if _, ok := r.gets[key]; ok { + return true + } + return false +} + +// Gstr . +func (r *HttpRequest) Gstr(key string) string { + if value, ok := r.gets[key]; ok { + return value + } + return "" +} + +// Gint . +func (r *HttpRequest) Gint(key string) int { + value := r.Gstr(key) + if value == "" { + return 0 + } + i, err := strconv.Atoi(value) + if err != nil { + return 0 + } + return i +} + +// Posts . +func (r *HttpRequest) Posts() map[string]string { + return r.posts +} + +// Phas . +func (r *HttpRequest) Phas(key string) bool { + if _, ok := r.posts[key]; ok { + return true + } + return false +} + +// Pstr . +func (r *HttpRequest) Pstr(key string) string { + if value, ok := r.posts[key]; ok { + return value + } + return "" +} + +// Pint . +func (r *HttpRequest) Pint(key string) int { + value := r.Pstr(key) + if value == "" { + return 0 + } + i, err := strconv.Atoi(value) + if err != nil { + return 0 + } + return i +} + +// Rint . +func (r *HttpRequest) Rint(key string) int { + if value := r.Pint(key); value != 0 { + return value + } + return r.Gint(key) +} + +// Rstr . +func (r *HttpRequest) Rstr(key string) string { + if value := r.Pstr(key); value != "" { + return value + } + return r.Gstr(key) +} + +// SetPosts . +func (r *HttpRequest) SetPosts(posts map[string]string) { + r.posts = posts +} + +// Cookies . +func (r *HttpRequest) Cookies() map[string]string { + return r.cookies +} + +// Chas . +func (r *HttpRequest) Chas(key string) bool { + if _, ok := r.cookies[key]; ok { + return true + } + return false +} + +// Cstr . +func (r *HttpRequest) Cstr(key string) string { + if value, ok := r.cookies[key]; ok { + return value + } + return "" +} + +// Cint . +func (r *HttpRequest) Cint(key string) int { + value := r.Cstr(key) + if value == "" { + return 0 + } + i, err := strconv.Atoi(value) + if err != nil { + return 0 + } + return i +} + +// IsKeepAlive . +func (r *HttpRequest) IsKeepAlive() bool { + return r.version == "HTTP/1.1" && r.Header("Connection") != "close" +} + +// Header . +func (r *HttpRequest) Header(key string) string { + if value, ok := r.headers[key]; ok { + return value + } + return "" +} + +// SetHeader . +func (r *HttpRequest) SetHeader(key string, value string) { + r.headers[key] = value +} + +// Var 。 +func (r *HttpRequest) Var(key string) interface{} { + return r.vars[key] +} + +// SetVar 。 +func (r *HttpRequest) SetVar(key string, value interface{}) { + r.vars[key] = value +} + +// Stream 。 +func (r *HttpRequest) Stream() []byte { + body := "" + if r.method == "POST" { + postsLen := len(r.posts) + if postsLen > 0 { + pairs := make([]string, 0, postsLen) + for key, value := range r.posts { + pairs = append(pairs, url.QueryEscape(key)+"="+url.QueryEscape(value)) + } + body = strings.Join(pairs, "&") + } + r.headers["Content-Length"] = strconv.Itoa(len(body)) + } + // @todo: 需要优化 + s := r.method + httpHeaderPartSeparator + r.uri + httpHeaderPartSeparator + r.version + httpHeaderSeparator + for key, value := range r.headers { + s += key + httpHeaderPropSeparator + " " + value + httpHeaderSeparator + } + s += httpHeaderSeparator + body + return []byte(s) +} + +// String 。 +func (r *HttpRequest) String() string { + return string(r.Stream()) +} + +// parseHairStream 。 +func (r *HttpRequest) parseHairStream(hairStream []byte) error { + parts := bytes.SplitN(hairStream, []byte(httpHeaderPartSeparator), 3) + if len(parts) != 3 { + return errors.New("bad http request") + } + r.method, r.uri, r.version = string(parts[0]), string(parts[1]), string(parts[2]) + r.pathInfo = r.uri + if markPos := strings.Index(r.uri, "?"); markPos != -1 { + r.pathInfo = r.uri[0:markPos] + if markPos != len(r.uri)-1 { + queryString := r.uri[markPos+1:] + kvStrings := strings.Split(queryString, "&") + for _, kvString := range kvStrings { + parts := strings.SplitN(kvString, "=", 2) + if len(parts) != 2 { + return errors.New("bad http request") + } + key, err := url.QueryUnescape(parts[0]) + if err != nil { + key = parts[0] + } + value, err := url.QueryUnescape(parts[1]) + if err != nil { + value = parts[1] + } + r.gets[key] = value + } + } + } + return nil +} + +func (r *HttpRequest) parseBodyStream(bodyStream []byte) error { + kvStrings := strings.Split(string(bodyStream), "&") + for _, kvString := range kvStrings { + parts := strings.SplitN(kvString, "=", 2) + if len(parts) != 2 { + return errors.New("bad http request") + } + key, err := url.QueryUnescape(parts[0]) + if err != nil { + key = parts[0] + } + value, err := url.QueryUnescape(parts[1]) + if err != nil { + value = parts[1] + } + r.posts[key] = value + } + return nil +} + +// HttpResponse . +type HttpResponse struct { + version string + code int + phrase string + headers map[string]string + bodyStream []byte +} + +var httpStatus = map[int]string{ + 200: "OK", + 301: "Moved Permanently", + 302: "Moved Temporarily", + 400: "Bad Request", + 401: "Unauthorized", + 403: "Forbidden", + 404: "Not Found", + 500: "Internal Server Error", +} + +// NewHttpResponse . +func NewHttpResponse() *HttpResponse { + return &HttpResponse{version: "HTTP/1.1", code: 200, phrase: httpStatus[200], headers: map[string]string{ + "Server": "Koala-Server", + "Content-Type": "text/html; charset=utf-8", + "Connection": "keep-alive", + }} +} + +// Version . +func (r *HttpResponse) Version() string { + return r.version +} + +// SetVersion . +func (r *HttpResponse) SetVersion(version string) { + r.version = version +} + +// Code . +func (r *HttpResponse) Code() int { + return r.code +} + +// SetCode . +func (r *HttpResponse) SetCode(code int) { + r.code = code + r.phrase = httpStatus[r.code] +} + +// Status . +func (r *HttpResponse) Status() (int, string) { + return r.code, r.phrase +} + +// SetStatus . +func (r *HttpResponse) SetStatus(code int, phrase string) { + r.code, r.phrase = code, phrase +} + +// Header . +func (r *HttpResponse) Header(key string) string { + if value, ok := r.headers[key]; ok { + return value + } + return "" +} + +// SetHeader . +func (r *HttpResponse) SetHeader(key string, value string) { + r.headers[key] = value +} + +// IsConnectionClose . +func (r *HttpResponse) IsConnectionClose() bool { + return r.Version() != "HTTP/1.1" || r.Header("Connection") == "close" +} + +// SetBodyStream . +func (r *HttpResponse) SetBodyStream(bodyStream []byte) { + r.bodyStream = bodyStream +} + +// BodyStream . +func (r *HttpResponse) BodyStream() []byte { + return r.bodyStream +} + +// SetBodyString . +func (r *HttpResponse) SetBodyString(bodyString string) { + r.SetBodyStream([]byte(bodyString)) +} + +// BodyString . +func (r *HttpResponse) BodyString() string { + return string(r.bodyStream) +} + +// Putb . +func (r *HttpResponse) Putb(stream []byte) { + if len(stream) > 0 { + r.bodyStream = append(r.bodyStream, stream...) + } +} + +// Puts . +func (r *HttpResponse) Puts(content string) { + r.bodyStream = append(r.bodyStream, content...) +} + +// Stream 。 +func (r *HttpResponse) Stream() []byte { + var b bytes.Buffer + b.WriteString(r.version) + b.WriteString(httpHeaderPartSeparator) + b.WriteString(strconv.Itoa(r.code)) + b.WriteString(httpHeaderPartSeparator) + b.WriteString(r.phrase) + b.WriteString(httpHeaderSeparator) + r.headers["Content-Length"] = strconv.Itoa(len(r.bodyStream)) + for key, value := range r.headers { + b.WriteString(key) + b.WriteString(httpHeaderPropSeparator + " ") + b.WriteString(value) + b.WriteString(httpHeaderSeparator) + } + b.WriteString(httpHeaderSeparator) + if len(r.bodyStream) > 0 { + b.Write(r.bodyStream) + } + return b.Bytes() +} + +func (r *HttpResponse) String() string { + return string(r.Stream()) +} + +func (r *HttpResponse) parseHairStream(hairStream []byte) error { + parts := bytes.SplitN(hairStream, []byte(httpHeaderPartSeparator), 3) + if len(parts) != 3 { + return errors.New("bad response") + } + r.version = string(parts[0]) + if code, err := strconv.Atoi(string(parts[1])); err == nil { + r.code = code + } else { + r.code = 500 + } + r.phrase = string(parts[2]) + return nil +} + +func (r *HttpResponse) parseBodyStream(bodyStream []byte) error { + r.bodyStream = bodyStream + return nil +} diff --git a/utility/log.go b/utility/log.go new file mode 100644 index 0000000..bedbdb2 --- /dev/null +++ b/utility/log.go @@ -0,0 +1,835 @@ +/** + * @file: log.go + * @package: main + * @author: heiyeluren + * @desc: Log operate file + * @date: 2013/6/24 + * @history: + * 2013/6/24 created file + * 2013/7/1 add logid function + * 2013/7/2 update code structure + * 2013/7/4 refactor all code + * 2013/7/10 add log_level operate + */ + +package utility + +import ( + "errors" + "fmt" + "io" + "math/rand" + "os" + "runtime" + "strconv" + "strings" + "sync" + "time" +) + +/* + +配置文件格式： +================================================= +#日志文件位置 (例：/var/log/koala.log) +log_notice_file_path = log/koala.log +log_debug_file_path = log/koala.log +log_trace_file_path = log/koala.log +log_fatal_file_path = log/koala.log.wf +log_warning_file_path = log/koala.log.wf + +#日志文件切割周期（1天:day; 1小时:hour; 10分钟:ten） +log_cron_time = day + +#日志chan队列的buffer长度，建议不要少于1024，#不建议多于102400，最长：2147483648 +log_chan_buff_size = 20480 + +#日志刷盘的间隔时间，单位:毫秒，建议500~5000毫秒(0.5s-5s)，建议不超过30秒 +log_flush_timer = 1000 +================================================= + +代码调用示例： + +import "github.com/heiyeluren/koala/utility/logger" +import "github.com/heiyeluren/koala/utility/network" + +logid := request.Header("WD_REQUEST_ID") //注意:只有问答产品才有WD_REQUEST_ID这个数据，其他服务按照对应id来 +log = logger.NewLogger(logid) //注意: 一个请求只能New一次，logid可以传空字符串，则会内部自己生成logid + +log_notice_msg := "[clientip=202.106.51.6 errno=0 errmsg=ok request_time=100ms]" +log.Notice(log_notice_msg) + +log_warning_msg := "[clientip=202.106.51.6 errno=101 errmsg=\"client is error\" request_time=45ms]" +log.Warning(log_warning_msg) + +*/ + +//======================== +// +// 外部调用Logger方法 +// +//======================== + +// Logger .Log 每次请求结构体数据 +type Logger struct { + LogID string +} + +//日志级别类型常量 +const ( + LogTypeFatal = 1 + LogTypeWarning = 2 + LogTypeNotice = 4 + LogTypeTrace = 8 + LogTypeDebug = 16 +) + +//日志类型对应信息 +const ( + LogTypeFatalStr = "FATAL" + LogTypeWarningStr = "WARNING" + LogTypeNoticeStr = "NOTICE" + LogTypeTraceStr = "TRACE" + LogTypeDebugStr = "DEBUG" +) + +// GLogTypeMap 日志信息map +var GLogTypeMap = map[int]string{ + LogTypeFatal: LogTypeFatalStr, + LogTypeWarning: LogTypeWarningStr, + LogTypeNotice: LogTypeNoticeStr, + LogTypeTrace: LogTypeTraceStr, + LogTypeDebug: LogTypeDebugStr, +} + +//------------------------ +// logger外部调用方法 +//------------------------ + +// NewLogger . 构造函数 +func NewLogger(logid string) *Logger { + return &Logger{LogID: logid} +} + +// Notice . +// 正常请求日志打印调用 +// 注意： +// 每个请求(request)只能调用本函数一次，函数里必须携带必须字段: ip, errno, errmsg 等字段，其他kv信息自己组织 +// 示例： +// Log_Notice("clientip=192.168.0.1 errno=0 errmsg=ok key1=valu2 key2=valu2") +func (l *Logger) Notice(logMessage string) { + l.syncMsg(LogTypeNotice, logMessage) +} + +// Trace . +// 函数调用栈trace日志打印调用 +func (l *Logger) Trace(logMessage string) { + l.syncMsg(LogTypeTrace, logMessage) +} + +// Debug . +// 函数调用调试debug日志打印调用 +func (l *Logger) Debug(logMessage string) { + l.syncMsg(LogTypeDebug, logMessage) +} + +// Fatal . +// 致命错误Fatal日志打印调用 +func (l *Logger) Fatal(logMessage string) { + l.syncMsg(LogTypeFatal, logMessage) +} + +// Warning . +// 警告错误warging日志打印调用 +func (l *Logger) Warning(logMessage string) { + l.syncMsg(LogTypeWarning, logMessage) +} + +//------------------------ +// logger内部使用方法 +//------------------------ + +// syncMsg . +// 写入日志到channel . +func (l *Logger) syncMsg(logType int, logMsg string) error { + //init request log + //Log_New() + + //从配置日志级别log_level判断当前日志是否需要入channel队列 + if (logType & GLogV.LogLevel) != logType { + return nil + } + + //G_Log_V := Log_New(G_Log_V) + if logType <= 0 || logMsg == "" { + return errors.New("log_type or log_msg param is empty") + } + + //拼装消息内容 + logStr := l.padMsg(logType, logMsg) + + //日志类型 + if _, ok := GLogTypeMap[logType]; !ok { + return errors.New("log_type is invalid") + } + + //设定消息格式 + logMsgData := LogMsgT{ + LogType: logType, + LogData: logStr, + } + + //写消息到channel + GLogV.LogChan <- logMsgData + + //判断当前整个channel 的buffer大小是否超过90%的阀值，超过就直接发送刷盘信号 + var threshold float32 + var currChanLen int = len(GLogV.LogChan) + threshold = float32(currChanLen) / float32(GLogV.LogChanBuffSize) + + if threshold >= 0.9 && !GFlushLogFlag { + GFlushLock.Lock() + GFlushLogFlag = true + GFlushLock.Unlock() + + GLogV.FlushLogChan <- true + //打印目前达到阀值了 + if LogIsDebug() { + LogDebugPrint(fmt.Sprintf("Out threshold!! Current G_Log_V.LogChan: %v; G_Log_V.LogChanBuffSize: %v", currChanLen, GLogV.LogChanBuffSize), nil) + } + } + + return nil +} + +// padMsg 拼装日志消息 +// 说明：主要是按照格式把消息给拼装起来 +// +// 日志格式示例： +// NOTICE: 2013-06-28 18:30:56 koala [logid=1234 filename=yyy.go lineno=29] [clientip=10.5.0.108 errno=0 errmsg="ok"] +// WARNING: 2013-06-28 18:30:56 koala [logid=1234 filename=yyy.go lineno=29] [clientip=10.5.0.108 errno=404 errmsg="json format invalid"] +func (l *Logger) padMsg(logType int, logMsg string) string { + + var ( + //日志拼装格式字符串 + logFormatStr string + logRetStr string + + //日志所需字段变量 + logTypeStr string + logDateTime string + logID string + logFilename string + logLineno int + logCallFunc string + + //log_clientip string + //log_errno int + //log_errmsg string + + //其他变量 + ok bool + fcName uintptr + ) + + //获取调用的 函数/文件名/行号 等信息 + fcName, logFilename, logLineno, ok = runtime.Caller(3) + if !ok { + errors.New("call runtime.Caller() fail") + } + logCallFunc = runtime.FuncForPC(fcName).Name() + + //展现调用文件名最后两段 + //println(log_filename) + + //判断当前操作系统路径分割符，获取调用文件最后两组路径信息 + osPathSeparator := LogGetOsSeparator(logFilename) + callPath := strings.Split(logFilename, osPathSeparator) + if pathLen := len(callPath); pathLen > 2 { + logFilename = strings.Join(callPath[pathLen-2:], osPathSeparator) + } + + //获取当前日期时间 (#吐槽: 不带这么奇葩的调用参数好不啦！难道这天是Go诞生滴日子??!!!#) + logDateTime = time.Now().Format("2006-01-02 15:04:05") + + //app name + //log_app_name = "koala" + + //logid读取 + logID = l.getLogID() + + //日志类型 + if logTypeStr, ok = GLogTypeMap[logType]; !ok { + errors.New("log_type is invalid") + } + + //拼装返回 + logFormatStr = "%s: %s [logid=%s file=%s no=%d call=%s] %s\n" + logRetStr = fmt.Sprintf(logFormatStr, logTypeStr, logDateTime, logID, logFilename, logLineno, logCallFunc, logMsg) + + //调试 + //println(log_ret_str) + + return logRetStr +} + +// getLogID 获取LogID +// 说明：从客户端request http头里看看是否可以获得logid，http头里可以传递一个：WD_REQUEST_ID +// 如果没有传递，则自己生成唯一logid +func (l *Logger) getLogID() string { + //获取request http头中的logid字段 + if l.LogID != "" { + return l.LogID + } + return l.genLogID() +} + +// genLogID 生成当前请求的Log ID +// 策略：主要是保证唯一logid，采用当前纳秒级时间+随机数生成 +func (l *Logger) genLogID() string { + //获取当前时间 + microTime := time.Now().UnixNano() + //生成随机数 + randNum := rand.New(rand.NewSource(microTime)).Intn(100000) + //生成logid：把纳秒时间+随机数生成 (注意：int64的转string使用 FormatInt，int型使用Itoa就行了) + //logid := fmt.Sprintf("%d%d", microTime, randNum) + return strconv.FormatInt(microTime, 10) + strconv.Itoa(randNum) +} + +//======================== +// +// 内部协程Run函数 +// +//======================== + +// LogMsgT 单条日志结构 +type LogMsgT struct { + LogType int + LogData string +} + +// LogT Log主chan队列配置 +type LogT struct { + + //------------------ + // Channel数据 + //------------------ + + //日志接收channel队列 + LogChan chan LogMsgT + + //是否马上日志刷盘: true or false，如果为true，则马上日志刷盘 (本chan暂时没有使用) + FlushLogChan chan bool + + //------------------ + // 配置相关数据 + //------------------ + + //所有日志文件位置 + LogFilePath map[int]string + + //日志文件位置 (例：/var/log/koala.log 和 /var/log/koala.log.wf) + LogNoticeFilePath string + LogErrorFilePath string + + //写入日志切割周期（1天:day、1小时:hour、15分钟：Fifteen、10分钟：Ten） + LogCronTime string + + //日志chan队列的buffer长度，建议不要少于1024，不多于102400，最长：2147483648 + LogChanBuffSize int + + //按照间隔时间日志刷盘的日志的间隔时间，单位：秒，建议1~5秒，不超过256 + LogFlushTimer int + + //------------------ + // 运行时相关数据 + //------------------ + + //去重的日志文件名和fd (实际需需要物理写入文件名和句柄) + MergeLogFile map[string]string + MergeLogFd map[string]*os.File + + //上游配置的map数据(必须包含所有所需项) + RunConfigMap map[string]string + + //是否开启日志库调试模式 + LogDebugOpen bool + + //日志打印的级别（需要打印那些日志） + LogLevel int + + // 日志文件的存在时间, 单位:天 + LogLifeTime int +} + +/** + * 配置项相关常量&变量 + */ +const ( + LogConfNoticeFilePath = "log_notice_file_path" + LogConfDebugFilePath = "log_debug_file_path" + LogConfTraceFilePath = "log_trace_file_path" + LogConfFatalFilePath = "log_fatal_file_path" + LogConfWarningFilePath = "log_warning_file_path" + + LogConfCronTime = "log_cron_time" + LogConfChanBuffSize = "log_channel_buff_size" + LogConfFlushTimer = "log_flush_timer" + LogConfDebugOpen = "log_debug_open" + LogConfLevel = "log_level" + LogConfFileLifeTime = "log_file_life_time" +) + +//配置选项值类型(字符串或数字) +const ( + LogConfTypeStr = 1 + LogConfTypeNum = 2 +) + +// GConfItemMap 配置项map全局变量 (定义一个选项输入的值是字符串还是数字) +var GConfItemMap = map[string]int{ + LogConfNoticeFilePath: LogConfTypeStr, + LogConfDebugFilePath: LogConfTypeStr, + LogConfTraceFilePath: LogConfTypeStr, + LogConfFatalFilePath: LogConfTypeStr, + LogConfWarningFilePath: LogConfTypeStr, + + LogConfCronTime: LogConfTypeStr, + LogConfChanBuffSize: LogConfTypeNum, + LogConfFlushTimer: LogConfTypeNum, + LogConfDebugOpen: LogConfTypeNum, + LogConfLevel: LogConfTypeNum, + LogConfFileLifeTime: LogConfTypeNum, +} + +// GConfFileToTypeMap 日志文件名与日志类型的映射 +var GConfFileToTypeMap = map[string]int{ + LogConfNoticeFilePath: LogTypeNotice, + LogConfDebugFilePath: LogTypeDebug, + LogConfTraceFilePath: LogTypeTrace, + LogConfFatalFilePath: LogTypeFatal, + LogConfWarningFilePath: LogTypeWarning, +} + +// GLogV .日志全局变量 +var GLogV *LogT + +// GOnceV 全局once +var GOnceV sync.Once + +// GFlushLogFlag 目前是否已经写入刷盘操作channel（保证全局只能写入一次，防止多协程操作阻塞） +var GFlushLogFlag bool = false + +//GFlushLock 控制 GFlushLogFlag 的全局锁 +var GFlushLock *sync.Mutex = &sync.Mutex{} + +/** +* 提供给协程调用的入口函数 +* +* @param RunConfigMap 是需要传递进来的配置信息key=>val的map数据 +*　调用示例： +* +//注意本调用必须在单独协程里运行 +m := map[string]string { + "log_notice_file_path": "log/koala.log" + "log_debug_file_path": "log/koala.log" + "log_trace_file_path": "log/koala.log" + "log_fatal_file_path": "log/koala.log.wf" + "log_warning_file_path": "log/koala.log.wf" + "log_cron_time": "day" + "log_chan_buff_size": "10240" + "log_flush_timer": "1" +} +go LogRun(m) +*/ + +// LogRun . 注意： 需要传递进来的配置是有要求的，必须是包含这些配置选项，否则会报错 +func LogRun(RunConfigMap map[string]string) { + //初始化全局变量 + if GLogV == nil { + GLogV = new(LogT) + } + + //设置配置map数据 + GLogV.RunConfigMap = RunConfigMap + + //调用初始化操作，全局只运行一次 + GOnceV.Do(LogInit) + + //启动log文件清理协程，定期删除过期的log文件 + go LogfileCleanup(int64(GLogV.LogLifeTime * 3600 * 24)) + + //永远循环等待channel的日志数据 + var logMsg LogMsgT + //var num int64 + for { + //监控是否有可以日志可以存取 + select { + case logMsg = <-GLogV.LogChan: + LogWriteFile(logMsg) + //if Log_Is_Debug() { + // Log_Debug_Print("G_Log_V.LogChan Length:", len(G_Log_V.LogChan)) + //} + default: + //breakLogChan长度 + //println("In Default ", num) + //打印目前G_Log_V的数据 + //if Log_Is_Debug() { + // Log_Debug_Print("G_Log_V.LogChan Length:", len(G_Log_V.LogChan)) + //} + time.Sleep(time.Duration(GLogV.LogFlushTimer) * time.Millisecond) + } + + //监控刷盘timer + //log_timer := time.NewTimer(time.Duration(G_Log_V.LogFlushTimer) * time.Millisecond) + select { + //超过设定时间开始检测刷盘（保证不会频繁写日志操作） + //case <-log_timer.C: + // log_timer.Stop() + // break + //如果收到刷盘channel的信号则刷盘且全局标志状态为 + case <-GLogV.FlushLogChan: + GFlushLock.Lock() + GFlushLogFlag = false + GFlushLock.Unlock() + + //log_timer.Stop() + break + default: + break + } + + } +} + +// LogInit . 初始化Log协程相关操作 +// 注意： 全局操作, 只能协程初始化的时候调用一次 +func LogInit() { + if GLogV.RunConfigMap == nil { + errors.New("Log_Init fail: RunConfigMap data is nil") + } + + //构建日志文件名和文件句柄map内存 + GLogV.LogFilePath = make(map[int]string, len(GLogTypeMap)) + + //判断各个配置选项是否存在 + for confItemKey := range GConfItemMap { + if _, ok := GLogV.RunConfigMap[confItemKey]; !ok { + fmt.Errorf("Log_Init fail: RunConfigMap not include item: %s", confItemKey) + } + } + + //扫描所有配置选项赋值给结构体 + var err error + var itemValStr string + var itemValNum int + for confItemK, confItemV := range GConfItemMap { + //对所有配置选项 进行类型转换 + if confItemV == LogConfTypeStr { + itemValStr = string(GLogV.RunConfigMap[confItemK]) + } else if confItemV == LogConfTypeNum { + if itemValNum, err = strconv.Atoi(GLogV.RunConfigMap[confItemK]); err != nil { + fmt.Errorf("Log conf read map[%s] fail, map is error", confItemK) + } + } + //进行各选项赋值 + switch confItemK { + //日志文件路径 + case LogConfNoticeFilePath: + GLogV.LogFilePath[LogTypeNotice] = itemValStr + case LogConfDebugFilePath: + GLogV.LogFilePath[LogTypeDebug] = itemValStr + case LogConfTraceFilePath: + GLogV.LogFilePath[LogTypeTrace] = itemValStr + case LogConfFatalFilePath: + GLogV.LogFilePath[LogTypeFatal] = itemValStr + case LogConfWarningFilePath: + GLogV.LogFilePath[LogTypeWarning] = itemValStr + + //其他配置选项 + case LogConfCronTime: + GLogV.LogCronTime = itemValStr + case LogConfChanBuffSize: + GLogV.LogChanBuffSize = itemValNum + case LogConfFlushTimer: + GLogV.LogFlushTimer = itemValNum + case LogConfDebugOpen: + if itemValNum == 1 { + GLogV.LogDebugOpen = true + } else { + GLogV.LogDebugOpen = false + } + case LogConfLevel: + GLogV.LogLevel = itemValNum + case LogConfFileLifeTime: + GLogV.LogLifeTime = itemValNum + } + } + + //设置日志channel buffer + if GLogV.LogChanBuffSize <= 0 { + GLogV.LogChanBuffSize = 1024 + } + GLogV.LogChan = make(chan LogMsgT, GLogV.LogChanBuffSize) + + //初始化唯一的日志文件名和fd + GLogV.MergeLogFile = make(map[string]string, len(GLogTypeMap)) + GLogV.MergeLogFd = make(map[string]*os.File, len(GLogTypeMap)) + for _, logFilePath := range GLogV.LogFilePath { + GLogV.MergeLogFile[logFilePath] = "" + GLogV.MergeLogFd[logFilePath] = nil + } + + //打印目前G_Log_V的数据 + if LogIsDebug() { + LogDebugPrint("G_Log_V data:", GLogV) + } + + // 设置清理时间不可为0 + if GLogV.LogLifeTime <= 0 { + GLogV.LogLifeTime = 7 // 默认7天 + } + +} + +// LogWriteFile . +// 写日志操作 +func LogWriteFile(logMsg LogMsgT) { + //读取多少行开始写日志 + //var max_line_num int + + //临时变量 + var ( + //动态生成需要最终输出的日志map + logMap map[string][]string + //读取单条的日志消息 + logMsgVar LogMsgT + //读取单个配置的日志文件名 + confFileName string + + writeBuf string + line string + ) + + //打开文件 + LogOpenFile() + + //初始化map数据都为 + logMap = make(map[string][]string, len(GConfFileToTypeMap)) + for confFileName = range GLogV.MergeLogFile { + logMap[confFileName] = []string{} + } + //fmt.Println(log_map) + + //压入第一条读取的日志(上游select读取的) + confFileName = GLogV.LogFilePath[logMsg.LogType] + logMap[confFileName] = []string{logMsg.LogData} + //fmt.Println(log_map) + + //读取日志(所有可读的日志都读取，然后按照需要打印的文件压入到不同map数组) + select { + case logMsgVar = <-GLogV.LogChan: + confFileName = GLogV.LogFilePath[logMsgVar.LogType] + logMap[confFileName] = append(logMap[confFileName], logMsgVar.LogData) + default: + break + } + //调试信息 + if LogIsDebug() { + LogDebugPrint("Log Map:", logMap) + } + + //写入所有日志(所有map所有文件的都写) + for confFileName = range GLogV.MergeLogFile { + if len(logMap[confFileName]) > 0 { + writeBuf = "" + for _, line = range logMap[confFileName] { + writeBuf += line + } + _, _ = GLogV.MergeLogFd[confFileName].WriteString(writeBuf) + _ = GLogV.MergeLogFd[confFileName].Sync() + + //调试信息 + if LogIsDebug() { + LogDebugPrint("Log String:", writeBuf) + } + } + } + +} + +// LogOpenFile . +// 打开&切割日志文件 +func LogOpenFile() error { + var ( + fileSuffix string + err error + confFileName string + runFileName string + newLogFileName string + newLogFileFd *os.File + ) + + //构造日志文件名 + fileSuffix = LogGetFileSuffix(time.Now()) + + //把重复日志文件都归一，然后进行相应日志文件的操作 + for confFileName, runFileName = range GLogV.MergeLogFile { + newLogFileName = fmt.Sprintf("%s.%s", confFileName, fileSuffix) + + //如果新旧文件名不同，说明需要切割文件了(第一次运行则是全部初始化文件) + if newLogFileName != runFileName { + //关闭旧日志文件 + if GLogV.MergeLogFd[confFileName] != nil { + if err = GLogV.MergeLogFd[confFileName].Close(); err != nil { + fmt.Errorf("Close log file %s fail", runFileName) + } + } + //初始化新日志文件 + GLogV.MergeLogFile[confFileName] = newLogFileName + GLogV.MergeLogFd[confFileName] = nil + + //创建&打开新日志文件 + newLogFileFd, err = os.OpenFile(newLogFileName, os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + fmt.Errorf("Open log file %s fail", newLogFileName) + } + newLogFileFd.Seek(0, io.SeekEnd) + + //把处理的相应的结果进行赋值 + GLogV.MergeLogFile[confFileName] = newLogFileName + GLogV.MergeLogFd[confFileName] = newLogFileFd + } + } + + //调试 + //fmt.Println(G_Log_V) + + return nil +} + +// LogGetFileSuffix . +// 获取日志文件的切割时间 +// 说明： +// 目前主要支持三种粒度的设置，基本这些粒度足够我们使用了 +// 1天:day; 1小时:hour; 10分钟:ten +func LogGetFileSuffix(now time.Time) string { + var fileSuffix string + //now := time.Now() + + switch GLogV.LogCronTime { + + //按照天切割日志 + case "day": + fileSuffix = now.Format("20060102") + + //按照小时切割日志 + case "hour": + fileSuffix = now.Format("20060102_15") + + //按照10分钟切割日志 + case "ten": + fileSuffix = fmt.Sprintf("%s%d0", now.Format("20060102_15"), int(now.Minute()/10)) + + //缺省按照小时 + default: + fileSuffix = now.Format("20060102_15") + } + + return fileSuffix +} + +// LogIsDebug . +// 获取目前是否是Debug模式 +func LogIsDebug() bool { + return GLogV.LogDebugOpen +} + +// LogDebugPrint . +// 日志打印输出到终端函数 +func LogDebugPrint(msg string, v interface{}) { + + //获取调用的 函数/文件名/行号 等信息 + fcName, logFilename, logLineno, ok := runtime.Caller(1) + if !ok { + errors.New("call runtime.Caller() fail") + } + logCallFunc := runtime.FuncForPC(fcName).Name() + + osPathSeparator := LogGetOsSeparator(logFilename) + callPath := strings.Split(logFilename, osPathSeparator) + if pathLen := len(callPath); pathLen > 2 { + logFilename = strings.Join(callPath[pathLen-2:], osPathSeparator) + } + + fmt.Println("\n=======================Log Debug Info Start=======================") + fmt.Println("[ call=", logCallFunc, "file=", logFilename, "no=", logLineno, "]") + if msg != "" { + fmt.Println(msg) + } + fmt.Println(v) + fmt.Println("=======================Log Debug Info End=======================\n") +} + +// LogGetOsSeparator . +// 获取当前操作系统的路径切割符 +// 说明: 主要为了解决 os.PathSeparator有些时候无法满足要求的问题 +func LogGetOsSeparator(pathName string) string { + //判断当前操作系统路径分割符 + var osPathSeparator = "/" + if strings.ContainsAny(pathName, "\\") { + osPathSeparator = "\\" + } + return osPathSeparator +} + +// LogfileCleanup . +// 对notice日志，进行定期清理，执行周期等同于“日志切割周期” +func LogfileCleanup(fileLifetime int64) { + + // println("clean up gorouting start!") + + // 5秒后再启动“清理”循环，错开启动初期的不稳定时段 + time.Sleep(time.Duration(5) * time.Second) + + var ( + // 清理周期，秒 + cycleTime int64 + // log文件保存周期，秒； 30天 + // file_lifetime int64 = 3600 * 24 * 30 + ) + + // 清理周期，设置为 “日志切割时间 ” + switch GLogV.LogCronTime { + case "day": + cycleTime = 3600 * 24 + case "hour": + cycleTime = 3600 + case "ten": + cycleTime = 600 + default: + cycleTime = 3600 + } + + // 目前，仅针对notice日志（所在log文件）进行删除操作 + confFileName := GLogV.LogFilePath[LogTypeNotice] + + // 删除log文件无限循环 + for { + var cleanupTime time.Time = time.Unix(time.Now().Unix()-fileLifetime, 0) + + // 计算出，待删除log文件名 + fileSuffix := LogGetFileSuffix(cleanupTime) + + // println(log_file_name) + + // 删除log文件 + // if err := os.Remove(log_file_name); err != nil { + // println(err.Error()) + // } + + os.Remove(fmt.Sprintf("%s.%s", confFileName, fileSuffix)) + // 等待下一个清理周期，sleep + time.Sleep(time.Duration(cycleTime) * time.Second) + } +} diff --git a/utility/tcp.go b/utility/tcp.go new file mode 100644 index 0000000..aecc370 --- /dev/null +++ b/utility/tcp.go @@ -0,0 +1,221 @@ +/** + * Koala Rule Engine Core + * + * @package: main + * @desc: koala engine tcp server + * + * @author: heiyeluren + * @github: https://github.com/heiyeluren + * @blog: https://blog.csdn.net/heiyeshuwu + * + */ + +package utility + +import ( + "encoding/gob" + "errors" + "net" + "time" +) + +// TcpListen 监听一个 tcp 地址 +func TcpListen(address string) (*TcpListener, error) { + tcpAddr, err := net.ResolveTCPAddr("tcp", address) + if err != nil { + return nil, err + } + tcpListener, err := net.ListenTCP("tcp", tcpAddr) + if err != nil { + return nil, err + } + return newTcpListener(tcpListener), nil +} + +// TcpListener tcp 监听器 +type TcpListener struct { + *net.TCPListener +} + +func newTcpListener(tcpListener *net.TCPListener) *TcpListener { + return &TcpListener{TCPListener: tcpListener} +} + +// Accept . +func (l *TcpListener) Accept() (*TcpConnection, error) { + tcpConn, err := l.TCPListener.AcceptTCP() + if err != nil { + return nil, err + } + return newTcpConnection(tcpConn), nil +} + +// TcpConnect 连接一个 tcp 地址 +func TcpConnect(localAddress string, remoteAddress string, timeout time.Duration) (*TcpConnection, error) { + // @todo: 本 api 调用不绑定 localAddress，可能会影响上层应用程序 + conn, err := net.DialTimeout("tcp", remoteAddress, timeout) + if err != nil { + return nil, err + } + tcpConn := conn.(*net.TCPConn) + return newTcpConnection(tcpConn), nil +} + +// TcpConnection tcp 连接 +type TcpConnection struct { + *net.TCPConn + protoBuffer []byte +} + +// 创建 tcp 连接对象 +func newTcpConnection(tcpConn *net.TCPConn) *TcpConnection { + return &TcpConnection{TCPConn: tcpConn} +} + +// ReadStream . +func (c *TcpConnection) ReadStream(stream []byte, count int) error { + if len(stream) < count { + return errors.New("bad stream") + } + stream = stream[:count] + left := count + for left > 0 { + n, err := c.Read(stream) + if n > 0 { + left -= n + if left > 0 { + stream = stream[n:] + } + } + if err != nil { + return err + } + } + return nil +} + +const protoBufferLen = 8 + +// ReadProtoBuffer . +func (c *TcpConnection) ReadProtoBuffer() error { + c.protoBuffer = make([]byte, protoBufferLen) + left := protoBufferLen + for left > 0 { + n, err := c.TCPConn.Read(c.protoBuffer) + if n > 0 { + left -= n + } + if err != nil { + return err + } + } + return nil +} + +// IsHttpProto . +func (c *TcpConnection) IsHttpProto() bool { + if c.protoBuffer == nil { + return false + } + if c.protoBuffer[0] == 'G' && c.protoBuffer[1] == 'E' && c.protoBuffer[2] == 'T' && c.protoBuffer[3] == ' ' && c.protoBuffer[4] == '/' { + return true + } + if c.protoBuffer[0] == 'P' && c.protoBuffer[1] == 'O' && c.protoBuffer[2] == 'S' && c.protoBuffer[3] == 'T' && c.protoBuffer[4] == ' ' && c.protoBuffer[5] == '/' { + return true + } + return false +} + +// Read . +func (c *TcpConnection) Read(p []byte) (n int, err error) { + if c.protoBuffer == nil { + n, err = c.TCPConn.Read(p) + } else { + pLen := len(p) + if pLen < protoBufferLen { + copy(p, c.protoBuffer[0:pLen]) + c.protoBuffer = c.protoBuffer[pLen:] + n = pLen + } else { + copy(p, c.protoBuffer) + c.protoBuffer = nil + n = protoBufferLen + } + } + return +} + +// ReadRpcRequest . +func (c *TcpConnection) ReadRpcRequest(timeout time.Duration) (*RpcRequest, error) { + if err := c.SetReadDeadline(time.Now().Add(timeout)); err != nil { + return nil, err + } + decoder := gob.NewDecoder(c) + request := NewRpcRequest() + if err := decoder.Decode(request); err != nil { + return nil, err + } + return request, nil +} + +// WriteRpcResponse . +func (c *TcpConnection) WriteRpcResponse(response *RpcResponse, timeout time.Duration) error { + if err := c.SetWriteDeadline(time.Now().Add(timeout)); err != nil { + return err + } + encoder := gob.NewEncoder(c) + if err := encoder.Encode(response); err != nil { + return err + } + return nil +} + +// Rpc . +func (c *TcpConnection) Rpc(request *RpcRequest, readTimeout time.Duration, writeTimeout time.Duration) (*RpcResponse, error) { + if err := c.SetWriteDeadline(time.Now().Add(writeTimeout)); err != nil { + return nil, err + } + encoder := gob.NewEncoder(c) + if err := encoder.Encode(request); err != nil { + return nil, err + } + if err := c.SetReadDeadline(time.Now().Add(readTimeout)); err != nil { + return nil, err + } + decoder := gob.NewDecoder(c) + response := NewRpcResponse() + if err := decoder.Decode(response); err != nil { + return nil, err + } + return response, nil +} + +// RegisterRpcTypeForValue 注册 rpc 类型 +func RegisterRpcTypeForValue(value interface{}) { + gob.Register(value) +} + +// RpcRequest . +type RpcRequest struct { + Func string + Args map[string]interface{} +} + +// NewRpcRequest . +func NewRpcRequest() *RpcRequest { + return &RpcRequest{Args: make(map[string]interface{})} +} + +// RpcResponse . +type RpcResponse struct { + Result bool + Reason string + Code int + Args map[string]interface{} + Data interface{} +} + +// NewRpcResponse . +func NewRpcResponse() *RpcResponse { + return &RpcResponse{Args: make(map[string]interface{})} +}