From d080306f288e23402bc0a16795a266186f5fec01 Mon Sep 17 00:00:00 2001 From: Shuhei Iitsuka Date: Mon, 18 Dec 2023 07:19:25 +0000 Subject: [PATCH 1/4] Add Thai model --- budoux/models/th.json | 1 + budoux/parser.py | 11 ++++ .../main/java/com/google/budoux/Parser.java | 9 +++ javascript/src/index.ts | 9 +++ javascript/src/webcomponents/budoux-th.ts | 33 ++++++++++ scripts/prepare_wisesight.py | 66 +++++++++++++++++++ setup.cfg | 1 + 7 files changed, 130 insertions(+) create mode 100644 budoux/models/th.json create mode 100644 javascript/src/webcomponents/budoux-th.ts create mode 100644 scripts/prepare_wisesight.py diff --git a/budoux/models/th.json b/budoux/models/th.json new file mode 100644 index 00000000..7f2cd6b2 --- /dev/null +++ b/budoux/models/th.json @@ -0,0 +1 @@ +{"UW3":{" ":4150,"เ":-5437,"น":1008,"ง":1122,"ั":-3075,"ะ":1494,"่":862,"ไ":-5122,"ย":868,"แ":-5772,"บ":630,"ห":-4445,"ค":-769,"ก":576,"า":127,"ด":964,"ี":310,"ม":725,"ิ":-1326,"้":66,"ส":-984,"์":1318,"ว":363,"ใ":-4337,"ๆ":3081,"โ":-3297,"ท":-846,"ุ":-1016,"(":1340,"-":1858,"a":-541,"ื":-721,"ข":-1009,"!":2180,"e":-550,"พ":-466,"​":-3387,"n":-628,"ช":-1071,".":1110,"i":-828,"?":1953,"0":194,"ู":111,"5":1080,"็":-1558,"ผ":-1945,"r":-393,"“":3487,"ฟ":-78,"t":-696,")":1982,"o":-277,"/":871,"4":899,"ㅠ":2154,"3":583,"—":2611,"ำ":-260,"…":1806,"‘":2391,"ร":-352,"\"":1837,"u":-501,"จ":49,"ฮ":-1244,"[":1891,"️":1355,"๊":-867,"+":801,"y":716,"╱":1265,"ซ":-636,"อ":-128,"k":-633,"%":476,"ป":-173,">":184,"s":-26,"C":-357,"ต":16,"_":-117,"R":-200,"S":-219,"9":304,"2":96,"ล":101,"N":-305,"m":178,"฿":691,"ฉ":-314,"1":-99,"’":175,"]":147},"UW4":{" ":4562,"า":-6029,"่":-6076,"เ":2104,"้":-6080,"ั":-5721,"ไ":1521,"ี":-6019,"ค":773,"ง":-2384,"แ":2044,"ะ":-4962,"น":-1344,"ย":-1586,"ิ":-4643,"ส":894,"ใ":2365,"ก":-300,"ท":388,"จ":330,"ๆ":4575,"ร":-611,"ข":449,"ว":-1269,"ห":878,"โ":999,"ู":-4855,"ุ":-3993,"ื":-4435,"ต":-177,"ป":520,"์":-3210,"ผ":1023,"พ":742,"!":3001,"อ":-347,"ช":313,"็":-2730,"ำ":-2716,")":2880,"e":-771,"ึ":-3515,"ด":-992,"-":2083,"ม":-1178,"a":-462,"1":1128,"?":2275,"ถ":248,"%":1299,"i":-945,"​":3044,"o":-961,"0":-1553,"2":1027,"n":-561,"”":4154,"5":612,"3":609,"ณ":-1619,"r":-560,"/":1206,"บ":-426,"—":655,"\"":2854,"’":1549,"ษ":-2136,"'":2061,"6":1094,"๊":-1961,"H":1334,"4":734,"ธ":-343,"l":-217,"S":1054,"8":-209,"๋":-1314,"u":-406,"+":610,"h":-228,"ㅠ":1070,"J":1989,"B":54,"<":2484,"ฮ":651,"ญ":-545,"]":1594,"*":1411,"A":733,"L":574,"G":369,"t":-185,"F":214,"x":-530,"z":-258,"d":-149,"=":883,"m":351,"&":-401,"9":-148,"P":-6,"U":-36,"ล":-82,"f":126,"w":-65,".":-59,"ฏ":-130,"T":14,"ภ":-17,"D":-14},"UW2":{" ":-743,"เ":-2180,"น":-263,"ั":509,"า":996,"ง":-694,"้":413,"ม":189,"อ":226,"ี":39,"ไ":753,"ิ":524,"ร":-233,"่":321,"ส":-575,"ด":590,"ุ":645,"แ":-485,"ใ":947,"โ":-1027,"ป":-133,"บ":-323,"็":347,"ะ":-517,"ต":528,"ค":518,"ว":573,"a":-339,"e":-461,"ก":-260,"ผ":1021,"n":-492,"0":402,"ข":311,"ซ":-495,"ึ":530,"ย":369,"i":-540,"1":273,"ล":-9,"t":-627,"ช":-134,"พ":304,"ภ":-58,"y":-504,"ถ":-103,"5":21,"ท":17,"ธ":498,"ู":404,"^":520,".":152,"S":-364,"ห":61,"l":-472,"w":-549,"ฮ":-218,"C":-894,"์":114,"8":-192,"m":270,"H":-352,"_":-98,"k":-419,"๋":788,"V":-133,"ศ":438,"T":-321,"J":-941,"D":-232,"!":221,"?":84,"p":-237,"P":10,"ฑ":559,"r":-23,"'":-208,"—":243,"A":-23},"UW5":{" ":-471,"ี":1819,"ั":1772,"า":1184,"เ":-512,"ะ":2097,"ก":-248,"ง":487,"ิ":1236,"อ":916,"ุ":1745,"ู":1878,"่":2005,"ไ":-951,"ำ":2511,"ล":331,"ร":470,"้":1409,"์":-3175,"ต":-516,"ค":-306,"จ":-843,"ึ":1641,"ป":144,"ื":1671,"บ":-164,"ว":671,"ส":-260,"e":-294,"ม":-329,"ข":-292,"แ":-551,"ห":-10,"5":480,"0":104,"a":-247,"ซ":-298,"ถ":287,"ๆ":-528,"ใ":-88,"ฮ":-566,"ฟ":-141,"2":-612,".":40,"r":-280,"ด":344,"h":526,"โ":-165,"1":411,"น":108,"R":-330,"C":-539,"็":858,"ผ":-436,"B":-951,"S":-126,"o":166,"z":-1265,"F":29,"x":-454,"-":250,"️":1030,"4":156,"๊":769,"ฉ":337,"ย":-113,"V":-196,"t":165,"_":-219,"ธ":-730,"%":-433,"3":-161,"9":137,"i":52,"ช":37,"๋":237,"/":163,",":-134,"ฏ":128,"ท":-8},"UW6":{"้":634,"่":397," ":-292,"า":198,"ง":235,"ก":439,"ี":-52,"ะ":635,"ว":350,"์":-2331,"a":-811,"บ":553,"น":-10,"ม":54,"e":-343,"0":373,"ิ":277,"็":397,"ื":473,"จ":-179,"ย":44,"i":-539,"โ":280,"ด":330,"r":-241,"ท":-108,"พ":-141,"ู":255,".":15,"ค":-139,"๊":1164,"D":-1206,"n":-264,"ช":-295,"ล":-101,"แ":-76,"ฟ":607,"ต":-105,"ไ":-195,"ั":227,"s":637,"ถ":-261,"เ":217,"อ":83,"ป":6,"l":-102,"(":-1050,"!":-600,"ฮ":-218,"ส":73,"8":-353,"ข":-88,"๋":1012,"ๆ":374,"ุ":223,"b":-315,"9":228,"o":145,"h":-34,"ฐ":314,"p":342,"2":-109,"V":314,"1":41,"ญ":-45,":":244,")":-20,"S":43,"ภ":-29,"U":34,"ร":2},"UW1":{"ไ":1349," ":-303,"แ":577,"อ":-206,"ใ":1868,"ี":-187,"ว":191,"ค":151,"ื":684,"ด":-205,"บ":-38,"ั":-33,"ก":404,"ล":185,"ถ":497,"ส":-137,"เ":454,"ห":587,"น":-93,"ะ":202,"ม":-111,"ึ":617,"ข":-379,"า":209,"ร":121,"ต":352,"e":-133,"จ":-365,"้":97,"พ":-126,"็":-235,"ฟ":646,"b":418,"d":-194,"5":19,"p":-491,"ภ":392,"ท":112,"ุ":-89,"t":-204,"โ":446,"-":-240,"ฮ":-66,".":79,"2":-406,"N":-531,"ำ":373,"ณ":-446,"u":-172,"3":28,"์":-62,"ย":113,"ง":121,"o":298,"J":-1640,"%":-1726,"╱":1033,"H":-266,"ซ":113,"9":-285,"B":-554,"l":-63,"i":77,"g":-144,"1":-76,"s":55,"ญ":96,"ๆ":-20,"a":5,"w":-20,"+":-23},"BW1":{"ี่":1653,"ระ":-2958,"ไป":2492,"่า":766,"ับ":1371,"มี":3306,"ก็":4237,"อง":389,"้า":675,"ือ":624,"รถ":2224,"ม่":998,"าก":367," (":3062,"จะ":3182,"เค":2007,"็น":565,"วย":270,"มา":1636,"ีย":-474,"ู่":625,"สี":1933,"สา":-1536,"่ะ":953,"ติ":819,"รอ":1575,"ิด":293,"ใน":2604,"ละ":1289,"วม":1377,"ไร":2849,"โห":5207,"คะ":1877,"ุก":1391,"ง ":1100,"ะ ":1939,"บ ":1953,"ยว":1037,"อย":188,"บบ":1163,"อก":316,"ดู":1129,"ื่":-1064,"า ":-132,"์ ":-394,"้น":-109,"ทะ":-772,"วา":-812,"ปี":311,"ตร":1002,"ิ ":-1642,"ั้":-915,"ขอ":1619,"าม":189,"ค ":-1448,"ทำ":1153,"นะ":2039,"ผม":1855,"ใจ":1095,"ัน":-36,"าร":463,"ิว":433,"เล":787,"นำ":1148,"วง":-827,"าส":-683,"สง":-1178,"ึก":1510,"าย":-226,"พอ":807,"สิ":1553,"โย":-690,"ภา":-1704,"ก ":1690,"ยา":-841,"จอ":947,"าง":-868,"กา":-468,"55":645,"ตะ":-2215,"็ม":-343,"แบ":-940,"คน":913,"กู":2108,"าด":-701,"มะ":-992,"อิ":353,"สะ":-1779,"อา":-1270,"กก":-52,"ล ":-1446,"ิ่":-631," '":2309,"้ย":366,"r ":-931,"ิป":262,"0 ":1903,"่อ":100,"อ ":-212,"สำ":-1026,"่ว":-108,"อบ":549,"ุ่":-628,"ั่":-568,"นน":938,"อน":277," -":1262,"วี":-1137,"วก":419,"าว":481,"ิต":478,"x ":-1575,"จำ":-66,"ที":557,"ัญ":-744,"รึ":1898,"กุ":1580,"ิก":431,"กะ":-787,"โต":1192,"ัพ":1006,"ยง":774,"ซี":-488,"ญ่":1048,"บอ":-300,"ด ":493,"อป":1387,"อค":-240,"^^":1407,"นก":-560,"มก":-388,"ส์":779,"ี ":424,"อล":-541,"จน":655,"ัส":791,"ัก":184,"บี":-1206,"9.":-1158,"เย":-989,"ห้":708,"ทน":561,"ก้":697,"โน":-402,"ใบ":-943,"ลน":-1696,"ัย":358,"่น":146,"ช้":453,"- ":642,"an":-493,"ยน":381,"ท์":-953,"าะ":523,"หา":214,"โล":499,"ดี":255,"แจ":355,"n ":-434,"nd":-1054,"a ":-140,"ูล":-390,"00":356,"ยู":-924,"าท":483,"ทด":-1251,"ัด":-420,"ออ":-72,"็ต":982," น":-422,"้อ":208,"คย":513,"ื้":-60,"คม":-593,"าอ":-73," ณ":-570,"ย์":-369,"ปก":-315,"t ":-574,"นู":689,"ัง":-14,"รค":271,"ัม":-352,"งง":594,"ัจ":-698," ":538,"ทา":375,"พล":-1025,"แด":-408,"็จ":96,"รี":-259,"วิ":-382,"คง":540,"V ":-581,"ีน":-99,"้ว":156,"ุส":1060,"ีป":-920,"็ป":-903,"A ":-530,"มค":-274,"ัท":561,"อต":274,"าา":327,"RT":921,"ay":641," T":-153,"he":-66,"mp":776,"็ค":92,"ิช":-704,"ใส":767,"งู":-868,"ผล":146,"_ ":-463,"ยุ":211,"มง":-580,"-V":-537,"ซน":-212,"ถา":-154,"สด":105,"าณ":416,"พร":-317,"ปน":363,"l ":189," 7":447,"ขา":36,"ทุ":-20,"ho":595,"กิ":-202,"..":110,"ชะ":-464,"็ท":-594,"ีด":316,"กำ":-254,"ัว":75," _":-577,"ษี":168,"บา":-167,"by":472,"11":327,"าศ":246,"จด":-390,"าช":-350,"e ":-142,"ซู":-260,"ลง":87,"่ ":27,"ai":112,"คา":112,"กด":72,"โร":-140,"นี":81,"ม้":123,"แก":137,"ูด":122," R":-166,"ดา":122,"ไพ":-190,"ชง":134,"าบ":68,"มส":-44,"ัต":-85,"มม":50,"พก":159,"งๆ":91,"ภู":-123,"ry":-81,"ไข":60,"ูง":17," 6":17,"ัล":2},"BW3":{"มา":2733,"อย":918,"ก็":3162,"รถ":3435,"ไร":-814,"ขอ":2848,"มี":3027,"อง":-1739,"ไป":2861,"ระ":-818,"นะ":2142,"เอ":-944,"น้":1779,"ลั":-2009,"นี":1348,"ยั":2016,"อน":-914,"คา":-745,"กั":1296,"หน":724,"ริ":-1904,"ว่":1633,"ซี":-982,"จะ":2647,"นิ":1812,"งา":2099,"เห":1624,"เป":358," เ":688,"คน":2113,"ลง":1162,"รอ":1660,"ดี":853,"ใจ":-1214,"มั":409," อ":-1112,", ":4155,"กก":-1518,"อา":1025," ว":-2435,"ก ":-972,"อิ":568,"หา":-510," แ":1486,"นน":-1496,"รี":-631,"ได":739,"อี":641,"ชม":1139,"ยก":1342,"ดู":1470,"ตอ":1747,"ผม":1816,"กล":1072,"น ":-559,"อก":-606,"ไห":68,"นา":570,"อั":1579,"กอ":-703,"ง่":2052,"ลา":-672,"พา":-240,"สา":193," ก":1321,"ทำ":738,"ขา":-213,"พน":1162,"% ":2267,"เว":-1286,"ลด":1795,"ยา":1243,"ยอ":625,"หม":830,"กค":-667," H":-653,"เด":18,"บ้":1231," ไ":1065,"สิ":-898,"..":1441,"อด":-1227,"ปี":697,"ปก":1568,"ติ":-680,"ห้":-231,"จั":-313,"วย":-550,"อบ":-261,") ":902,"ภั":-1613,"เน":-900,"สม":256,"ผั":-2123,"สุ":-968,"เต":-213,"เล":-393,"จู":-2325,"เพ":89,"! ":1698,"ถ้":1433," ป":-362,"รั":-288," ส":432,"พื":-118," ท":1068,"ร้":115,"ขน":1074,"จน":1988,"ตี":-166,": ":2475,"ลย":1156,"กเ":-423," A":-1276,"เท":-319,"ขั":-400,"น่":842,"เฟ":-540,"ยิ":1793,"โด":298,"ก่":1640,"แด":-341,"ตล":1358,". ":835,"ทา":440,"กำ":2634,"เเ":987,"แบ":-573,"หล":523,"อ่":1576,"ข่":-1571,"2 ":-590,"มว":520,"วว":-1423,"ชา":-329,"ละ":1044,"20":806,"นเ":-287,"ก้":188,"ทั":764," -":539,"นั":837,"นึ":1314,"พบ":948,"เข":303," N":-1153,"ยย":-599,"รร":-288,"วอ":1033,"กว":364,"ลี":-919,"กด":846,"08":1194,"บุ":175,"เย":89,"มอ":533,"สั":-169,"ซื":951,"บา":884,"BF":2410,"วิ":582,"รส":604,"คง":685,"ตร":-134,"โป":842,"ถึ":-395," บ":-299,"พล":375,"กป":-576,"ใช":393," S":-650,"แป":-368,"’ ":1306," ใ":662,"งั":1340,"ดั":292,"ต่":554,"ดา":354,"คว":526,"ยว":-224," ":405," M":-513,"ตก":1075,"เม":-74,"โย":-209,"เช":112,"แค":344," U":-1135,"ธี":-560,"พอ":199,"ฉ่":-1237,"พั":-379,"บท":709,"ผล":357,"ดด":-842,"ถน":393,"รว":101,"ใน":519,"ปอ":-759," ณ":-1180,"ออ":-89,"ตะ":-964,"สน":429,"พุ":-562,"ภา":755,"ชอ":662,"ร่":-34,"6 ":194,"แส":216,"ต้":322,"กุ":779,"กู":783,"ๆ ":538,"ลพ":1407,"รึ":491,".เ":821,"คุ":336,"ฟอ":764,"แต":492,"ณ ":817,"s ":-751,"โน":-335,"Ra":-533," R":-216,"55":528,"ไซ":-695,"บ่":738,"คั":373,"เร":-47,"ปป":-380,"แพ":-275,"แจ":278,"มุ":-685,"วะ":636," ค":241,"แท":430,"Pr":-481,"ช่":279,"ตบ":956,"เธ":960,"ผึ":-831,"อื":369," ข":192,"ถา":-22,"งอ":-141,"ชิ":-107,"ส่":-292,"โพ":98,"1 ":-324,"พว":514," c":155,"((":-673,"แห":62,"88":-304,"วี":91," พ":-85,"ชน":243,"อุ":244,"ยู":295,"He":247,"Tร":-544,"กิ":239,"สี":111,"ดร":45,"ม่":-223,"มะ":19,"? ":297,"2ค":348,"ฉบ":159,"ช้":-80,"ปล":-37,"ใค":179,"m ":233,"ลุ":-129," :":190," ต":30,"งี":68,"ยุ":170,"' ":115,"หั":-73,"ฟี":-186,"นู":153,"คิ":70," x":-44,"กา":8,"ปร":8,"พญ":85,"5 ":-17,"ไฟ":49," P":-26," (":31,"จี":-29,"กม":-26,"นุ":20,"พร":5,"ยน":-11,"มิ":11,",0":-8,"8 ":11,"บน":2,"ตึ":-5},"TW1":{"นี้":2638,"ว่า":2251,"ที่":1040,"ล้ว":1847,"การ":531,"เลย":2064,"เรา":3293,"ของ":1669,"ราย":-2282,"แต่":1263,"ได้":1162,"อีก":3076,"ท่า":-701,"้า ":-1483,"ไม่":901,"่อย":1295,"กิน":801,"ื้อ":1061,"รือ":2746,"เอา":2656,"คือ":2825,"มัน":354,"ั้ง":-420,"คุณ":1341,"าติ":1941,"ป็น":862,"่อง":-513,"าขา":1816,"ขอบ":-981,"ชอบ":2000,"าคา":1281,"ยู่":1372,"มเค":2174,"สัน":-484,"ประ":-1440,"ะเล":1815,"ามา":-748,"นัก":-1979,"วัน":-494,"าร์":-963,"ยาก":1761,"บอก":2862,"ค่า":1355,"พี่":973,"box":3019,"da ":-855,"ีป ":-2784,"แม่":-1965,"วลา":1132,"และ":803,"ึงเ":2238,"ยี่":-1653,"้วย":821,"กับ":467,"ด้า":-528,"ราม":-1580,"ข้า":383,"จ้า":-704,"เก็":-2219,"เทศ":688,"หัว":-842,"บาท":1762,"ฒนา":1199,"อิน":-581," ณ ":-1447,"อ็ม":-888,"ต้น":-1414,"้าง":741,"หม่":865,"รรค":1449,"ช็ค":2674,"ือน":1321,"ั่ง":503," ปี":1887,"ตัว":13,"ีโอ":1912,"ค่ะ":1621,"ัตร":1152,"ฮอน":-1096,"่าย":1185,"ต์ ":-870,"ถ้า":579,"้อน":1306,"ภาค":2027,"ยละ":-1000,"ญี่":-2789,"หร่":1141,"ลิป":1204,"โรง":-1567,"สัง":-1340,"ร์ท":1362,"ั้ย":700,"จัด":561,"ชัย":-1866,"ด็ก":1540,"ปิด":1134,"ไหม":1072,"ื่อ":699,"งาน":541,"ออส":1618,"ลูก":-414,"จาก":851,"ิกา":1330," มี":-345,"สนอ":1002,"ราง":-782,"ผิว":354,"ออก":494,"แระ":2025,"สมา":-1583,"่าง":862,"เจอ":950,"้อม":579,"กระ":-511,"ุ่ง":-1037,"หนา":1024," มา":196,"ต่ง":-783,"ปัญ":-595,"รับ":227,"ไหน":915,"an ":-502,"ะนำ":677,"สัก":677,"ทรู":624,"ขาว":1155,"ียน":258,"ิช ":-1281,"ุรี":1416,"ป็ด":1272,"ต้า":698,"ค้า":571,"เธอ":707,"ควร":898,"เบา":1449,"่าน":918,"ัคร":1176,"สีย":603,"ตึง":-835,"ัน ":-384,"ปปป":923,"มัย":600,"ั้น":495," อะ":-1111,"รค์":-1154,"กัน":85,"ty ":-818,"อเค":1450,"ปาก":-118,"ีโค":802,"ส่ง":-376,"เหน":-509,"พวง":-1338,"ทาง":1089,"ิอิ":954,"ตาม":402,"นมา":257,"ป้ง":-155,"ม่ง":911,"มไป":-533,"ญหา":483,"ยอะ":630,"ถาน":-1002,"ตรง":795,"คัด":-313,"กาง":-541,"ตี้":960,"าน ":231,"ริส":478,"นาด":854,"ภาย":-635,"ื่ม":960,"ลาย":692,"กัด":795,"โปร":269,"ว่ะ":-243,"สาร":314,"ถไฟ":1068,"ครง":-744,"ชน ":-1140,"แค่":310,"นชน":627,"ิ่ง":135,"นนน":476,"ตรา":513,"บแจ":620,"ยบก":1182,"อก็":-339,"่อน":41,"ถึง":101,"ยัง":239,"he ":-273,"คอน":-423,"้าอ":-333,"แพ้":178,"าตร":-907,"ษณา":803,"ภาพ":585," รบ":905,"ั่น":170,"ดัด":-591,"ฟรี":508,"ทีย":-607,"สวย":298,"ุ้น":490,"nd ":-761,"ผู้":-138,"เสา":1005,"มิต":-819,"ียด":219,"ร็จ":264," PA":711,"ี้ ":-322,"กษา":539,"ด้ว":-255,"ะสบ":-519,"The":-122,"นาม":-494,"ะชา":-486,"มาย":295,"ลยี":551,"ต้ม":-758,"ั่ว":507,"แดด":349,"พลส":601,"ผัส":198,"ูมิ":375,"คับ":364,"ร์ ":42,"ิ้น":-50,"ย T":-570," ไป":-306,"นัด":471," งง":430,"สดุ":414,"ดาว":252,"nda":87,"ขี้":-259,"ือ ":-365,"ลัง":-95," พอ":358,"ew ":-249,"มอง":303,"้าย":299,"แดก":487,"ถาม":183,"กติ":224,"าง ":191,"ติด":120,"ติบ":-280,"พวก":236,"วี ":-646," สน":365,"กล้":436,"ใช้":146,"ปดู":184,"จ้ง":105,"ะดา":225,"ยยย":147,"กอะ":432,"ทัก":420,"ก็บ":149,"มวย":277,"ลอง":209,"อัน":61,"โสม":154,"งสี":267," ทำ":-190,"เวน":-321,"ลัว":37,"ข็ง":-170,"คืน":275," ชา":396,"้กะ":374,"คาด":-232,"นอง":-222,"โทร":246,"ะยะ":-223,"็อก":-104,"แดง":-209,"นดี":150,"่ดี":194,"บโอ":371,"นับ":-131,"พัน":-176,"ปลง":72,"ชิบ":-226,"พอต":270,"ูแล":109,"ดิม":115,"า 2":186,"นท์":-18,"san":-97,"ะใจ":-182,"ุ้ม":52,"ขวา":117,"มมม":141,"ทอน":-122,"ยิน":-105,"ลิก":37,"าแฟ":61,"ทร ":-247,"ิตร":58,"ท็ก":-29,"มอก":93,"เอก":-95,"คัน":37,"ดี้":-72,"กซ์":-31,"ี่ย":-5,"พระ":72,"กชม":72,"ก่า":66,"รรม":11,"ตพอ":-40,"หาด":-23,"55 ":-23,"กตุ":31,"นลี":46,"มพู":14,"ยัด":34,"All":-17,"คตะ":-23,"เอม":-20,"ร่า":8,"ter":5},"BW2":{"่อ":-1298,"คร":-1523,"ะเ":-1312,"อย":-2542,"55":-5954,"าก":-566,"กล":-1600,"้อ":-1186,"ปร":-1494,"าเ":386,"งเ":-798," ":-3208,"นท":-686,"าค":-295,"์ ":-1456,"มเ":-1180,"น ":-1065,"นแ":-591,"อง":-1361,"..":-4184,"าร":-332,"้ว":-510,"อน":-464,"ีย":-1846,"ือ":-1519,"รถ":-1765,"งแ":-227,"นร":1024,"ๆๆ":-3943,"อก":-436,"กต":-1456,"ิอ":1211,"อบ":-774,"าอ":305,"กร":-1040," ป":-1335,"นเ":-709," จ":-1612,"มพ":-1453,"นไ":-1141,"นก":929,"าน":-487,"่น":-219,"้เ":381,"ป ":-1225,"่ง":-757,"รอ":-476,"่ว":-1014,"ตร":-1559,"ีโ":-819,"งร":669,"นม":1418,"พร":-1502," เ":-612,"ปล":-1207,"าม":-41,"ดเ":-1013,"าข":-229,"นต":-300,"ยค":736,"้ ":-892,"งห":-460,"มก":565,"่ส":-488," อ":-1303,"กว":-1001,"ีเ":-891,"0น":2663,"มข":-567,"งอ":481,"วย":-364,"งก":-187,"ยอ":-203,"บเ":-146,"กส":579," บ":-910,"งพ":-257,"่เ":35,"อม":-310,"งม":758,"ำเ":-1065,"าย":-529,"0 ":1199,"บผ":-1891,"งย":695,"้จ":-656,"y ":-949,"คว":-762,"ิก":-438,"ูแ":-784,"งด":707,"ยต":-1023,"ูก":-405,"ล ":-1276,"นล":321," ม":927," H":-1751,"อว":426,"าแ":189,"ำห":-433," 1":329,"ปป":-1367," ว":-1750,"!!":-3030,"งข":-349,"ทร":-595,"้ร":1116,"บป":-533,"าท":-537,"ึก":-1330,"มย":863,"ปไ":-140," พ":-781,"))":-3430,"ำต":448,"งบ":877,"์เ":-573," 2":-865,"พน":-1420," N":-695,"วก":-324,"็ม":-559,"งล":521,"ิด":-159," ส":-270,"ีม":-205,"ีพ":-1228,"ะม":415,"ดด":-83,"กน":296,"นด":-195,"มน":443,"1 ":675,"2 ":487,"าพ":495," S":-1482,"นข":-577,"A ":-452,"วส":585,"ตล":-248,"งส":-56,"V ":-190,"มค":-587,"ถน":-857,"้ไ":-408," A":-1170,"ำม":-280,"ค ":-331," c":495,"มล":665," G":499,"01":-728,"x ":-628,"คล":-482,"าบ":-33,"งไ":-31,"อท":152,"ุก":-120,"ดง":741,"้a":993,"นห":-320,"ูว":337,"กอ":343," R":-62,"ิท":800," 0":920,"ีถ":1228,"l ":382,"รส":200,"56":-1388,"้ก":151,"ยร":380," C":-127,"า.":523,"ะต":-494," e":379,"ณภ":-808,"สโ":950,"บร":-249,"ลอ":-373,"อข":701,"ูส":503,"นศ":963,"นใ":-171,"นน":-288,"้ม":-91,"วเ":-91,"มช":-356,"ิไ":530,"วพ":572," o":638,".3":-380,"ิเ":161,"มโ":-645,"ล/":938,"r ":-279,"ยโ":447,"อเ":-181,"ต.":499," (":300,"งน":-147,"รป":680,"ม3":1283,"คอ":-296,"ณด":418,"สน":36,"สเ":153,"์ไ":362,"ีน":269,"ณอ":349,"กไ":350,"ง5":488,"ำใ":-507,"??":-1052,"ยพ":-108,"1ห":568,"รค":-353,"ะย":-624,"กถ":-863,"จด":-427,"มส":201,"คห":663,"งจ":-99,"ตไ":459,"2ห":437,"กโ":349,"ิว":-184,"5+":-532,"ิส":-170,"ม2":620,"วผ":-509,"ะท":-171," E":274,"มม":-29,"์ย":292,"ำท":302,"มผ":-283,"ลเ":-138,"ะV":181," a":125,"ิ ":-137,"นช":170,"าa":330,":)":-465,"ูด":-269,"์.":129,"ตบ":398,"g ":176,"าถ":84,"่แ":68,"0%":-302,"ลโ":104,"งต":-119,"25":-113,"รแ":110,"eเ":166," b":79,".น":124,"งท":-32,": ":58,"น3":123," M":-38,"ิร":29,"นจ":26,"าซ":43,"้โ":79,"++":-92,"คก":34,"ตพ":-87,"ุอ":49,"นว":-29,"งโ":-31,"มแ":-14,"สว":-28,"่ล":31,"0ป":-43,"่ใ":23,"ดห":17,"งง":-11,"คโ":-5,"์พ":14,"อส":-2},"TW4":{"ครั":2075,"ไม่":4224,"กิน":1709,"วัน":2095,"ด้ว":3007,"สัน":-3192,"กว่":2157,"ค้า":-2173,"อยู":3493,"อยา":3496,"ราค":1803,"ชั่":-2019,"ขี่":-927,"ยัง":1823,"ไหม":2377,"ไลน":-1832,"ลยล":4508,"สด้":-3516,"มาก":821,"ที่":671,"ต้อ":830,"ไหน":1182,"ไหร":-677,"จริ":1637,"นคร":2337,"แล้":1152,"นาน":1305,"ฟ้า":-1555,"ใหญ":-1024,"การ":974,"ผิด":-1146,"เลย":877,"อะไ":1910,"สวย":1677,"เท่":2288,"เดี":1602,"ทุน":-1141,"น่า":648,"ให้":755,"กัน":131,"สุด":-178,"ง ร":3219,"เตอ":-1512,"กระ":2014,"มั้":2114,"ราย":1759,"ออก":1040,"เคร":1805,"ทุก":1596,"มัน":996," 2":-753,"ค่ะ":1302,"เป็":750,"ด้า":-116,"รวม":1379,"หรื":2075,"น.":148,"ค่า":-790,"ฟรี":1380,"ตรง":1923,"มาล":-967,"งวด":2435,"รี่":-987,"ประ":195,"555":702," ปา":-1831,"อีก":1265,"พาน":-690,"ทะเ":-789,"บริ":1144,"รัฐ":808,"เนอ":-674,"โตโ":1843,"ปรุ":-1353,"และ":1119,"กลั":874,"บอก":1690,"เทพ":-743,"นาค":-2126,"อย่":1289,"ดอก":2133,"ว่า":274,"มวย":1209,"รสช":1646,"แบบ":-444,"มึง":1247,"หน่":601,"แปล":-865,"ติด":1599,"ไทย":256,"เอา":1247,"อร่":1382,"ตี้":-823,"มาร":-713,"อนา":1059,"เวล":1572,"หรอ":1196,"กัด":-1651," GD":-1535,"เบี":397,"ทรั":-1145,"บีก":-824,"ตาม":1075,"ไปไ":-1756,"เสร":-178," ปล":-354,"รีโ":2072,"ดิน":-37,"ท้า":-1464,"เจ้":816,"สิน":-774,"เยอ":1190,"เพื":1316,"ขวด":1195,"the":1724,"มอง":756,"น้อ":1008,"ระเ":-536,"โดย":1196," แฮ":-1453,"ผิว":767,"เจี":-1658,"หวย":-1666,"ก้อ":791,"จัก":-842,"ภาพ":-1022,"ในว":-1216,"มี่":-1709,"หมา":-1096,"มัย":-328,"ยอด":601," อว":-1837,"เรา":315,"นี้":555,"เพล":-346," มี":375,"กับ":335,"นุ่":1285,"สาย":912," Pr":-1155,"บุห":1006,"สวิ":-797,"กติ":868,"อาย":666,"จาก":692,"ยกโ":1496," ไว":-1118,"ก็อ":-1218,"เคื":-1154,"ระห":579,"จัด":739,"วรร":1100," Ho":-1065,"กรี":1135,"รุ่":584,"ลิป":384,"นะค":491,"เรี":-466,"ชมพ":1157,"ขนา":1047,"คะ":804,"อสั":980,"เด่":-154,"นอก":896,"เพร":762,"ลาด":-495,"!!":1197,"ทรู":886,"เฟ่":-636,"ลิต":693,"โคร":965,"ยี่":853,"ม้า":-1249,"ทอง":-809," เอ":-300,"งี้":1044,"ไว้":496,"ดาว":143,"คือ":469,"เนื":696,"สาร":-877,"กรุ":567,"เที":236,"แม่":524,"รูป":348,"เพิ":553,"ผลิ":583,"ขวั":-942,"พีเ":-615,"ถนน":874,"นอน":783,"สิว":548," 20":561,"Ref":-423,"หน้":56,"ตัว":286,"ลอง":417,"กรร":-286,"นะเ":-429,"ใจม":510,"บทเ":817,"ดีก":505,"นาว":414,"มีแ":-557,"ร่ว":397,"แรง":-220," ตั":228,"ต้ว":680,"หาร":-171," 3":-150,"อะ":538,"สูง":-358,"บัต":352,"อิน":508,"อเม":532,"ธรร":313,"หนอ":-244," วอ":-748,"ริม":571,"รัก":-326,"บาท":447,"สบา":242,"เวอ":-441,"ปกต":560,"วัย":673,"ละเ":-339,"เร็":-246,"นี่":304,"เขา":264,"เซ็":117,"ดูด":-539,"กด":280,"ใจค":58,"ในแ":-979,"เพช":-511,"ดีไ":742,"รพ.":662,"อาก":-414,"ตอน":301,"ใจก":-17,"เบิ":-464,"ยาร":117,"2 ม":-519,"เหม":118,"อมช":497,"เด็":-278,"เล็":236,"ไรบ":-389,"แสง":361,"ปาก":365,"Cos":-701,"ดี้":-387,"กลิ":443,"ปิด":224,"เขี":478,"มกร":454,"ตาย":515,"สวั":345,"ตรว":509,"!!!":370,"เต้":-390,"ดัง":154,"โล/":374,"หลอ":274," พา":-374,"วีอ":227,"10ป":-488," บ้":-170,"หาด":218,"พัฟ":-532,"ออง":-339,"ทั้":289,"แพง":345,"นำเ":183,"ไร้":313,"ปี":203,"สือ":-418,"ใจแ":-203,"แบน":319,"เมื":154,"มาค":-156,"ศรี":36,"ออน":129,"ยกเ":386,"กด้":-299,"เรน":-161,"กาน":137," เพ":125,"ทาน":-92,"ตลา":57,") โ":-291,"ใจท":258,"อัน":148,"มอล":240,"รีฟ":136,"นาท":210,"50":160,"คนี":-233,"เค้":156,"201":112,"เลี":141,"อาท":99,"พลั":91,"นวั":70,"นิส":82,"เกอ":-91," บา":-44," จร":-105,"เนิ":-79,"อุด":49,"ณ บ":163,"ทอม":102,"สัก":61,"สุข":-49," เจ":-90,"พาไ":61,"Hon":58,"ตอบ":101,"อาห":55,"Fac":63,"เฟส":43,"ดิอ":37,"ปรั":14,"คุณ":20,"เป้":-31," Ra":-31,"โทน":34,"ๆๆ":46,"เปอ":-46,"บาง":14,"ซูซ":25,"นิด":20,"ภาย":11,"ดีค":-8,"ลง":-14,"มะ":2,"หมอ":2},"TW2":{"อะไ":-1876,"อีก":-1242,"ับข":-2138,"้นท":-1939,"ี่ย":-2409,"ี่ ":-2022,"ต์ ":-651,"ล้ว":-920,"้า ":-985,"ลงท":-611,"า ซ":-2637,"อบค":-1243,"งเช":1943,"ราค":-1285,"ู้ส":-2080,"มาก":-988,"การ":-635,"อ โ":-3253,"ัวเ":-394,"นะน":-1869,"ร์ช":-1118,"ง ก":-3299,"da ":-693,"น เ":-1356,"ยงใ":-2542,"ังไ":-1533,"ับเ":-1396,"่มข":-1161,"วยส":-847,"ายพ":-581,"องก":-444,"้าข":-386,": ":-994,"ขนส":-1595,"ด้ท":-2422,"้ ฮ":-2074,"างเ":-159,"้าง":-684,"ย ต":-2005,"้ำม":-1133,"้าม":1351,"่งข":-1281,"ิดถ":-1124,"ไป ":-2021,"วนต":-1074,"ต่ท":-1281,"้าร":569,"สนใ":1491,"ง ร":-1381,"ังน":-1316,"ทะเ":-1128,"ากม":-1372,"่าง":-883,"ัวห":-1299,"์ ธ":-1306,"น จ":-1968,"ิดต":-1717,"ช้ไ":-1639,"วยเ":-343,"ty ":-661,"มาเ":-1047,"มาร":-377,"ินน":-710,"ร์ ":458,"้ๆๆ":-1134,"นาย":1248,"้วแ":-1308,"55 ":-1107,"น บ":-1155,"ร ส":-1121,"่อม":-289,"โคค":821,"ท เ":-1153,"ันม":-1089,"อกเ":-919,"ันท":-343,". .":-1481,"างไ":-1152,"์ ไ":-855,"ๆๆๆ":-470,"่อก":450,"อกแ":-600,"้งฝ":-800,"ปีแ":-1277,"ินเ":-589,"บอก":-984,"ยวก":-1088,"ด้แ":-938,"อิอ":1094,"าวบ":-1729,"ายไ":-479,"้อค":-952,"ักง":-583,"อนเ":-405,"หนอ":-276,"น น":-1146,"ุดน":-1097,"่าน":-411,"ค แ":-1364,"ัวบ":1092,"ม่ว":-746,"ิดท":576,"ิตน":633,"างก":-367,"องจ":-400,"ัดผ":-633,"ขออ":872,"้นห":-512,"ังส":-490,"อมเ":-365,"่ะ.":-864,"าดใ":-382,"านใ":-624,"โตโ":-528,"ากบ":-1332,"ินข":-820,"องม":456,"ันส":159,"บนเ":967,"ู้ใ":-624,"ห้ม":-1027,"ูลค":-582,"ช้จ":-950,"่าต":-513,"ห้ ":-386,"ัสด":-745,"ไปก":-601,"อบถ":-324,"่นเ":600,"ี่ม":397,"วนห":-337,"คนข":-872,"้าไ":292,"ดีเ":364,"ัวว":493,"ร์ต":-318,"ุดย":-753,"่งส":373,"ึกว":415,"อดเ":453,"้นเ":106,"ูกก":340,"88-":-399,"่งเ":-311,"้าท":-303,"ิวเ":-335,"ามส":-285,"้งว":482,"ินส":-278,"มาส":-219,"่มเ":-259,"ึงเ":-319,"ินไ":356,"่งป":-417,"ี้ส":-414,"ีน ":-259,"ยนใ":-282,"ขอบ":-298,"ุกว":-474,"บบน":-208,"ารท":-254,"ังป":-390,"็นต":-517,"่อไ":-210,"ันใ":234,"มะเ":-121,"ไมม":-467,"สีเ":219,"ุกอ":232,"สีส":-254,"วยง":-384,"บนซ":-253,"5 ป":-324,"ปีใ":-193,"ัวใ":-184,"้าใ":-148,"อมร":-270,"ัยเ":-387,"ุปป":409,"าม ":-92,"ัล ":-197,"จำเ":-270,"ละน":208,"อกก":-101,"ลาง":166,"ราบ":-100,"อกห":150,"ันถ":-143,"ารจ":-99,"ูกต":-105,"อะเ":26,"ลงร":-169,"อบอ":125,"องโ":92,"ง ว":-92,"ตาข":110,"in ":40,"้นต":63,"อนแ":34,"ึงข":-66,"้วย":-17,"e H":-11,"นนน":-25,"้ H":-17,"่อส":-5},"TW3":{"กว่":-2147,"นแด":-1440,"ครั":-928,"งแต":-1264,"ำไม":-1913,"ารถ":-3032,"่อย":-1015,"บผิ":-338,"ะไร":-1028,"ะนำ":-1363,"นที":-689,"งไง":-308,"กระ":-700,"นนี":-940,"าคา":-85,"งพื":-1661,"วเอ":-1287,". .":-3092," ไว":-2116,"งสำ":-2535,"าเล":-962,"อร่":-1480,"งเท":-849,"นคร":-474,"นไป":-652,"่สุ":-1902,"น น":-1686,"งกา":-329,"งนี":-422,"ดขา":-1004,"ถไฟ":-2197," รู":-1711," ปา":-1703,"ะเอ":-1174,"กค้":-695,"ดชอ":-2140,"ยสู":-1263,"กซี":-1115,"่ว่":-1653," ม":-1513,"งหน":-197,"่ทำ":-1892,"น บ":-1717,"ูแล":-1546,"า ซ":-1568," เอ":-687,"นเน":-183,"ปได":-2685,"5 ป":-1096,"่ดิ":-1787,"าขอ":-1408,"านา":1547,"บปร":-349,"์ ธ":-1107,"ีแม":-1417,"งปร":-624,"บสน":-1654,"าเด":-1017,"ยละ":-1014,"บริ":-258,"ี ค":-1546,"่ห้":-1288,"นไห":1424,"าตร":-1466,"้ว่":727,"าสุ":-917,"ากิ":464,"งกิ":-1064,"บอา":-1198,"น เ":-983,"ปกิ":-817," ฮา":-898,"ยเห":-1495,"ยเป":-1362,"ฟเฟ":-593,"ีมา":-1147," กั":-1315,"ยแล":-1045,"็มี":-1016,"ายน":968,"ถนะ":-643,"กพน":-1509,"กร้":1201,"ง ว":-1097,"นรา":-1660," Ho":-593,"ดเด":-741,"อเพ":-794,"อยา":-186,"ีเอ":-542,"งออ":-853," จร":-1161," สป":-770,"อออ":-542,"ิกา":-397,"มที":-885,"รเม":-1040,"บนี":-806,"้ถึ":-691,"นตี":-503,"กก้":808,"นเข":-879,"าเต":-594,"กกก":-493," สว":-954,"าจอ":-488," วอ":-823,"อ โ":-565," เฮ":-448," หน":-647,"อสั":-755,"งกั":-583,"งงา":-396,"งหั":-792,"งทะ":-800,"งนั":-489,"รแพ":-490,"ณค่":-1088,"ณสม":-987," บ้":-814,"วคื":-1045,"ดนี":-108,"ย ต":-694,"งคื":-522,"าร่":-503,"งชื":-678,"นดู":-278,"นใจ":-243,"้แบ":-167,"วแต":-466,"อรั":338,"3. ":563,"นคะ":-423,"าแด":-594,"รตา":-293,"นมา":-459,"บคร":-515,"นนา":419,"งจา":-428,"าชิ":-284,"ยพั":-412,"มเค":-96,"อสม":394,"เชื":145," พา":-552,"วไห":-351,"นสุ":-337,"์สิ":-198,"มยอ":155," บา":-346,"ตวิ":194,"อเจ":-356,"อเส":327,"งดื":-482,"ลเล":-74,"อมั":-269,"งเด":296,"น จ":-165,"าออ":229,"บเป":-307,"อกา":6,"ยหล":-279,"งน้":-250,"ะสิ":-268,"มพา":-158,"มเม":-83,"ำเข":-278,"ะเว":-194,"าวั":146,"อยก":229,"ค แ":-200," Sh":-200,"นพิ":-199," วง":170,"e H":-145,"นแส":-168,"ากั":52,"กบ้":-146,"มแป":-120,"อบา":82,"น10":-67,"าตี":49,"นนั":79,"วอย":-155,"ีวิ":-108,"าง ":84,"าน้":84,"งขา":-41,"มสุ":-90,"ล บ":-49,"ลกา":93,"งรา":-90,"นเร":52,"งเว":34,"้มั":-37,"จริ":-8,"กลิ":-28,"าร้":8,"d O":-2}} diff --git a/budoux/parser.py b/budoux/parser.py index 1ff642b8..c11babe9 100644 --- a/budoux/parser.py +++ b/budoux/parser.py @@ -133,3 +133,14 @@ def load_default_traditional_chinese_parser() -> Parser: with open(os.path.join(MODEL_DIR, 'zh-hant.json'), encoding='utf-8') as f: model = json.load(f) return Parser(model) + + +def load_default_thai_parser() -> Parser: + """Loads a parser equipped with the default Thai model. + + Returns: + A parser (:obj:`budoux.Parser`). + """ + with open(os.path.join(MODEL_DIR, 'th.json'), encoding='utf-8') as f: + model = json.load(f) + return Parser(model) diff --git a/java/src/main/java/com/google/budoux/Parser.java b/java/src/main/java/com/google/budoux/Parser.java index 74f5fd69..66f7f62e 100644 --- a/java/src/main/java/com/google/budoux/Parser.java +++ b/java/src/main/java/com/google/budoux/Parser.java @@ -84,6 +84,15 @@ public static Parser loadDefaultTraditionalChineseParser() { return loadByFileName("/models/zh-hant.json"); } + /** + * Loads the default Thai parser. + * + * @return a BudouX parser with the default Thai model. + */ + public static Parser loadDefaultThaiParser() { + return loadByFileName("/models/th.json"); + } + /** * Loads a parser by specifying the model file path. * diff --git a/javascript/src/index.ts b/javascript/src/index.ts index 8f4081e4..b3009456 100644 --- a/javascript/src/index.ts +++ b/javascript/src/index.ts @@ -17,6 +17,7 @@ import {model as jaModel} from './data/models/ja.js'; import {model as zhHansModel} from './data/models/zh-hans.js'; import {model as zhHantModel} from './data/models/zh-hant.js'; +import {model as thModel} from './data/models/th.js'; import {HTMLProcessingParser} from './html_processor.js'; export {Parser} from './parser.js'; @@ -47,6 +48,13 @@ export const loadDefaultTraditionalChineseParser = () => { return new HTMLProcessingParser(zhHantModel); }; +/** + * Loads a parser equipped with the default Thai model. + * @returns A parser with the default Thai model. + */ +export const loadDefaultThaiParser = () => { + return new HTMLProcessingParser(thModel); +}; /** * Loads available default parsers. * @returns A map between available lang codes and their default parsers. @@ -56,5 +64,6 @@ export const loadDefaultParsers = () => { ['ja', loadDefaultJapaneseParser()], ['zh-hans', loadDefaultSimplifiedChineseParser()], ['zh-hant', loadDefaultTraditionalChineseParser()], + ['th', loadDefaultThaiParser()], ]); }; diff --git a/javascript/src/webcomponents/budoux-th.ts b/javascript/src/webcomponents/budoux-th.ts new file mode 100644 index 00000000..9b972076 --- /dev/null +++ b/javascript/src/webcomponents/budoux-th.ts @@ -0,0 +1,33 @@ +/** + * @license + * Copyright 2023 Google LLC + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {loadDefaultThaiParser} from '../index.js'; +import {BudouXBaseElement} from './budoux-base.js'; + +/** + * BudouX Thai Web component. + */ +export class BudouXThaiElement extends BudouXBaseElement { + /** + * BudouX Thai Web component constructor. + */ + constructor() { + super(); + this.parser = loadDefaultThaiParser(); + } +} + +customElements.define('budoux-th', BudouXThaiElement); diff --git a/scripts/prepare_wisesight.py b/scripts/prepare_wisesight.py new file mode 100644 index 00000000..cbc57cc4 --- /dev/null +++ b/scripts/prepare_wisesight.py @@ -0,0 +1,66 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Prepares a dataset from the Wisesight corpus. + +Before running this script, you need to download the Wisesight corpus by running: + +$ curl -o wisesight-1000-samples-tokenised.label https://raw.githubusercontent.com/PyThaiNLP/wisesight-sentiment/master/word-tokenization/wisesight-1000-samples-tokenised.label +$ curl -o wisesight-160-samples-tokenised.label https://raw.githubusercontent.com/PyThaiNLP/wisesight-sentiment/master/word-tokenization/wisesight-160-samples-tokenised.label + +Then run this command as follows over each file. + +$ python scripts/prepare_wisesight.py wisesight-1000-samples-tokenised.label -o source_train.txt +$ python scripts/prepare_wisesight.py wisesight-160-samples-tokenised.label -o source_val.txt +""" +import argparse +import re + +import regex + + +def parse_args() -> argparse.Namespace: + DEFAULT_OUT_PATH = 'source.txt' + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + 'source_filepath', help='Path to a Wisesight corpus label file.') + parser.add_argument( + '-o', + '--outfile', + help=f'File path to the output dataset. (default: {DEFAULT_OUT_PATH})', + default=DEFAULT_OUT_PATH) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + source_filepath = args.source_filepath + target_filepath = args.outfile + + with open(target_filepath, 'w') as outfile: + with open(source_filepath) as infile: + for line in infile: + line = line.strip() + line = re.sub(r'https?://[^ ]+', '', line) # Remove URLs + line = re.sub(r'#[^ ]+', '', line) # Remove hashtags + line = regex.compile(r'\p{Emoji_Presentation=Yes}+').sub( + '', line) # Remove emojis + line = re.sub(r'\|+', '|', line) # Remove consecutive separators + line = re.sub(r'(\|\s)*\|$', '', line) # Remove redundant spaces + outfile.write(line.replace('|', '▁') + '\n') # Replace the separators. + print('\033[92mTraining data is output to: %s\033[0m' % (target_filepath)) + + +if __name__ == '__main__': + main() diff --git a/setup.cfg b/setup.cfg index 7c9c640e..31af4ae2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,6 +28,7 @@ dev = isort mypy==1.7.1 pytest + regex toml twine types-setuptools From ccd21c1fc981310272d41c08cca712e675222fd8 Mon Sep 17 00:00:00 2001 From: Shuhei Iitsuka Date: Mon, 18 Dec 2023 07:43:41 +0000 Subject: [PATCH 2/4] Update demo with Thai model --- demo/package-lock.json | 4 ++-- demo/src/app.ts | 4 +++- demo/static/index.html | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/demo/package-lock.json b/demo/package-lock.json index 047920fe..a0f90bb9 100644 --- a/demo/package-lock.json +++ b/demo/package-lock.json @@ -25,7 +25,7 @@ "license": "Apache-2.0", "dependencies": { "commander": "^11.0.0", - "jsdom": "^22.1.0" + "jsdom": "^23.0.0" }, "bin": { "budoux": "bin/budoux.js" @@ -1377,7 +1377,7 @@ "eslint-plugin-prettier": "^5.0.0", "jasmine": "^5.0.2", "jasmine-core": "^5.0.1", - "jsdom": "^22.1.0", + "jsdom": "^23.0.0", "karma": "^6.4.2", "karma-chrome-launcher": "^3.2.0", "karma-jasmine": "^5.1.0", diff --git a/demo/src/app.ts b/demo/src/app.ts index 293713f1..77dd2f05 100644 --- a/demo/src/app.ts +++ b/demo/src/app.ts @@ -14,17 +14,19 @@ * limitations under the License. */ -import { loadDefaultJapaneseParser, loadDefaultSimplifiedChineseParser, loadDefaultTraditionalChineseParser } from 'budoux'; +import { loadDefaultJapaneseParser, loadDefaultSimplifiedChineseParser, loadDefaultTraditionalChineseParser, loadDefaultThaiParser } from 'budoux'; const parsers = new Map([ ['ja', loadDefaultJapaneseParser()], ['zh-hans', loadDefaultSimplifiedChineseParser()], ['zh-hant', loadDefaultTraditionalChineseParser()], + ['th', loadDefaultThaiParser()] ]); const defaultInputs = new Map([ ['ja', 'Google の使命は、世界中の情報を整理し、世界中の人がアクセスできて使えるようにすることです。'], ['zh-hans', '我们的使命是整合全球信息，供大众使用，让人人受益。'], ['zh-hant', '我們的使命是匯整全球資訊，供大眾使用，使人人受惠。'], + ['th', 'พันธกิจของเราคือการจัดระเบียบข้อมูลในโลกนี้และทำให้เข้าถึงได้ง่ายในทุกที่และมีประโยชน์'] ]) const inputTextElement = document.getElementById('input') as HTMLTextAreaElement; const outputContainerElement = document.getElementById('output') as HTMLElement; diff --git a/demo/static/index.html b/demo/static/index.html index 9620e265..21584934 100644 --- a/demo/static/index.html +++ b/demo/static/index.html @@ -74,6 +74,7 @@

BudouX 🍇

+

From ae6de43d1874358dd9f507af61a5ce5241601e60 Mon Sep 17 00:00:00 2001 From: Shuhei Iitsuka Date: Tue, 19 Dec 2023 04:38:10 +0000 Subject: [PATCH 3/4] Install types-regex --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 31af4ae2..cb38173a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,6 +31,7 @@ dev = regex toml twine + types-regex types-setuptools yapf From 604ee99d855992edb207b1a126ac169bef505b9c Mon Sep 17 00:00:00 2001 From: Shuhei Iitsuka Date: Tue, 19 Dec 2023 04:40:50 +0000 Subject: [PATCH 4/4] nit fix --- scripts/prepare_wisesight.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prepare_wisesight.py b/scripts/prepare_wisesight.py index cbc57cc4..3baa9501 100644 --- a/scripts/prepare_wisesight.py +++ b/scripts/prepare_wisesight.py @@ -1,4 +1,4 @@ -# Copyright 2021 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.