diff --git a/.gitmodules b/.gitmodules index 1a1a4c8388..1514e200e4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -515,6 +515,9 @@ [submodule "vendor/grammars/hoon-grammar"] path = vendor/grammars/hoon-grammar url = https://github.com/pkova/hoon-grammar.git +[submodule "vendor/grammars/ide-tools"] + path = vendor/grammars/ide-tools + url = https://github.com/toitware/ide-tools.git [submodule "vendor/grammars/idl.tmbundle"] path = vendor/grammars/idl.tmbundle url = https://github.com/mgalloy/idl.tmbundle diff --git a/grammars.yml b/grammars.yml index a5407a73eb..213b0a6d3b 100644 --- a/grammars.yml +++ b/grammars.yml @@ -431,6 +431,8 @@ vendor/grammars/holyc.tmbundle: - source.hc vendor/grammars/hoon-grammar: - source.hoon +vendor/grammars/ide-tools: +- source.toit vendor/grammars/idl.tmbundle: - source.idl - source.idl-dlm diff --git a/lib/linguist/languages.yml b/lib/linguist/languages.yml index 7b3ece948d..b235fa2414 100644 --- a/lib/linguist/languages.yml +++ b/lib/linguist/languages.yml @@ -7093,6 +7093,14 @@ Thrift: - ".thrift" ace_mode: text language_id: 374 +Toit: + type: programming + color: "#c2c9fb" + extensions: + - ".toit" + tm_scope: source.toit + ace_mode: text + language_id: 356554395 Turing: type: programming color: "#cf142b" diff --git a/samples/Toit/chatbot.toit b/samples/Toit/chatbot.toit new file mode 100644 index 0000000000..f7f43148cd --- /dev/null +++ b/samples/Toit/chatbot.toit @@ -0,0 +1,177 @@ +// Copyright (C) 2023 Florian Loitsch +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +import openai + +/** +If there is a gap of more than MAX_GAP between messages, we clear the +conversation. +*/ +MAX_GAP ::= Duration --m=3 +/** The maximum number of messages we keep in memory for each chat. */ +MAX_MESSAGES ::= 20 + +class TimestampedMessage: + text/string + timestamp/Time + is_from_assistant/bool + + constructor --.text --.timestamp --.is_from_assistant: + +/** +A base class for a chat bot. + +In addition to implementing the abstract methods $my_name_, $send_message_, + subclasses must periodically call $clear_old_messages_. Ideally, this + should happen whenever a new event is received from the server. + +Typically, a `run` function proceeds in three steps: +``` +run: + while true: + message := get_new_message // From the chat server. + clear_old_messages_ // Call to this bot. + store_message_ message.text --chat_id=message.chat_id + if should_respond: // This might depend on the message or client. + request_response_ message.chat_id +``` + +The chat_id is only necessary for bots that can be in multiple channels. +It's safe to use 0 if the bot doesn't need to keep track of multiple chats. + +Once the bot receives a response it calls $send_message_. +*/ +abstract class ChatBot: + // The client is created lazily, to avoid memory pressure during startup. + openai_client_/openai.Client? := null + openai_key_/string? := ? + + // Maps from chat-id to deque. + // Only authenticated chat-ids are in this map. + all_messages_/Map := {:} + + /** + Creates a new instance of the bot. + + The $max_gap parameter is used to determine if a chat has moved on to + a new topic (which leads to a new conversation for the AI bot). + + The $max_messages parameter is used to determine how many messages + are kept in memory for each chat. + */ + constructor + --openai_key/string + --max_gap/Duration=MAX_GAP + --max_messages/int=MAX_MESSAGES: + openai_key_ = openai_key + + close: + if openai_client_: + openai_client_.close + openai_client_ = null + openai_key_ = null + + /** The name of the bot. Sent as a system message. */ + abstract my_name_ -> string + + /** Sends a message to the given $chat_id. */ + abstract send_message_ text/string --chat_id/any + + /** Returns the messages for the given $chat_id. */ + messages_for_ chat_id/any -> Deque: + return all_messages_.get chat_id --init=: Deque + + /** + Drops old messages from all watched chats. + Uses the $MAX_GAP constant to determine if a chat has moved on to + a new topic (which leads to a new conversation for the AI bot). + */ + clear_old_messages_: + now := Time.now + all_messages_.do: | chat_id/any messages/Deque | + if messages.is_empty: continue.do + last_message := messages.last + if (last_message.timestamp.to now) > MAX_GAP: + print "Clearing $chat_id" + messages.clear + + /** + Builds an OpenAI conversation for the given $chat_id. + + Returns a list of $openai.ChatMessage objects. + */ + build_conversation_ chat_id/any -> List: + result := [ + openai.ChatMessage.system "You are contributing to chat of potentially multiple people. Your name is '$my_name_'. Be short.", + ] + messages := messages_for_ chat_id + messages.do: | timestamped_message/TimestampedMessage | + if timestamped_message.is_from_assistant: + result.add (openai.ChatMessage.assistant timestamped_message.text) + else: + // We are not combining multiple messages from the user. + // Typically, the chat is a back and forth between the user and + // the assistant. For memory reasons we prefer to make individual + // messages. + result.add (openai.ChatMessage.user timestamped_message.text) + return result + + /** Stores the $response that the assistant produced in the chat. */ + store_assistant_response_ response/string --chat_id/any: + messages := messages_for_ chat_id + messages.add (TimestampedMessage + --text=response + --timestamp=Time.now + --is_from_assistant) + + /** + Stores a user-provided $text in the list of messages for the + given $chat_id. + The $text should contain the name of the author. + */ + store_message_ text/string --chat_id/any --timestamp/Time=Time.now -> none: + messages := messages_for_ chat_id + // Drop messages if we have too many of them. + if messages.size >= MAX_MESSAGES: + messages.remove_first + + new_timestamped_message := TimestampedMessage + // We store the user with the message. + // This is mainly so we don't need to create a new string + // when we create the conversation. + --text=text + --timestamp=timestamp + --is_from_assistant=false + messages.add new_timestamped_message + + /** + Sends a response to the given $chat_id. + */ + send_response_ chat_id/any: + if not openai_client_: + if not openai_key_: throw "Closed" + openai_client_ = openai.Client --key=openai_key_ + + conversation := build_conversation_ chat_id + response := openai_client_.complete_chat + --conversation=conversation + --max_tokens=300 + store_assistant_response_ response --chat_id=chat_id + send_message_ response --chat_id=chat_id diff --git a/samples/Toit/logger.toit b/samples/Toit/logger.toit new file mode 100644 index 0000000000..70261c7bb5 --- /dev/null +++ b/samples/Toit/logger.toit @@ -0,0 +1,150 @@ +// Copyright (C) 2023 Florian Loitsch +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +import certificate_roots +import gpio +import monitor +import uart +import supabase + +// After 5s offload the data. Even if the device is not quiet. +MAX_OFFLOAD_DELAY ::= Duration --s=5 +// Offload if we accumulate more than 2kb of data. +MAX_BUFFERED_DATA ::= 2000 +// Offload if we have not received any data for 500ms. +MAX_QUIET_FOR_OFFLOAD ::= Duration --ms=500 + +LOGS_TABLE ::= "logs" + +class LogForwarder: + pin_/gpio.Pin + port_/uart.Port + buffered_/List := [] // Of ByteArray. + buffered_size_/int := 0 + offload_task_/Task? := null + upload_/Lambda + + constructor pin_number/int --upload/Lambda: + pin_ = gpio.Pin pin_number + port_ = uart.Port --rx=pin_ --tx=null --baud_rate=115200 + upload_ = upload + offload_task_ = task:: + offload_ + + close: + if offload_task_: + offload_task_.cancel + port_.close + pin_.close + offload_task_ = null + + listen: + while true: + chunk := port_.read + buffer_ chunk + + buffer_ data/ByteArray: + print "Received $data.to_string_non_throwing" + buffered_.add data + buffered_size_ += data.size + if buffered_size_ > MAX_BUFFERED_DATA: + offload_ + + offload_: + last_offload := Time.now + while true: + last_message := Time.now + old_size := buffered_size_ + while (Duration.since last_message) < MAX_QUIET_FOR_OFFLOAD: + sleep --ms=20 + + if buffered_size_ == 0: + // Reset the timer. + last_offload = Time.now + last_message = last_offload + continue + + if (Duration.since last_offload) > MAX_OFFLOAD_DELAY: + break + + if buffered_size_ == old_size: + continue + + if buffered_size_ > MAX_BUFFERED_DATA: + print "too much data" + break + + last_message = Time.now + old_size = buffered_size_ + + print "Offloading" + total := ByteArray buffered_size_ + offset := 0 + buffered_.do: + total.replace offset it + offset += it.size + to_upload := total.to_string_non_throwing + buffered_.clear + buffered_size_ = 0 + print "Uploading: $to_upload" + upload_.call to_upload + +main + --supabase_project/string + --supabase_anon/string + --device_id/string + --pin_rx1/int + --pin_rx2/int?: + + client/supabase.Client? := null + forwarder1/LogForwarder? := null + forwarder2/LogForwarder? := null + + while true: + // Trying to work around https://github.com/toitlang/pkg-http/issues/89 + catch --trace: + client = supabase.Client.tls + --host="$(supabase_project).supabase.co" + --anon=supabase_anon + --root_certificates=[certificate_roots.BALTIMORE_CYBERTRUST_ROOT] + + mutex := monitor.Mutex + + offload := :: | uart_pin/int data/string | + mutex.do: + client.rest.insert --no-return_inserted LOGS_TABLE { + "device_id": device_id, + "uart_pin": uart_pin, + "data": data, + } + + if pin_rx2: + task:: + forwarder2 = LogForwarder pin_rx2 --upload=:: | data/string | + offload.call pin_rx2 data + forwarder2.listen + + forwarder1 = LogForwarder pin_rx1 --upload=:: | data/string | + offload.call pin_rx1 data + forwarder1.listen + + if forwarder1: forwarder1.close + if forwarder2: forwarder2.close + if client: client.close diff --git a/samples/Toit/one_wire.toit b/samples/Toit/one_wire.toit new file mode 100644 index 0000000000..2c5521af05 --- /dev/null +++ b/samples/Toit/one_wire.toit @@ -0,0 +1,39 @@ +// Copyright (C) 2023 Toitware ApS +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +// FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +import gpio +import one_wire +import one_wire.family as one_wire + +DATA_PIN ::= 32 + +main: + bus := one_wire.Bus (gpio.Pin 32) + print "Listing all devices on bus:" + bus.do: + family_id := one_wire.family_id --device_id=it + print " $(%x it): $(one_wire.family_to_string family_id)" + + print "Listing only ds18b20 devices on bus:" + // Only list ds18b20 devices. + bus.do --family=one_wire.FAMILY_DS18B20: + print " $(%x it)" + + print "Demonstrating how to skip families." + // Skip families. + bus.do: + family_id := one_wire.family_id --device_id=it + print " Got called with id: $(%x it) - $(one_wire.family_to_string family_id)" + if family_id == one_wire.FAMILY_DS18B20: + print " Skipping remaining devices of this family." + continue.do one_wire.Bus.SKIP_FAMILY diff --git a/vendor/README.md b/vendor/README.md index 2f5a08d98b..57aad933bd 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -551,6 +551,7 @@ This is a list of grammars that Linguist selects to provide syntax highlighting - **Texinfo:** [Alhadis/language-texinfo](https://github.com/Alhadis/language-texinfo) - **TextMate Properties:** [textmate/textmate.tmbundle](https://github.com/textmate/textmate.tmbundle) - **Thrift:** [textmate/thrift.tmbundle](https://github.com/textmate/thrift.tmbundle) +- **Toit:** [toitware/ide-tools](https://github.com/toitware/ide-tools) - **Turing:** [Alhadis/language-turing](https://github.com/Alhadis/language-turing) - **Turtle:** [peta/turtle.tmbundle](https://github.com/peta/turtle.tmbundle) - **Twig:** [Anomareh/PHP-Twig.tmbundle](https://github.com/Anomareh/PHP-Twig.tmbundle) diff --git a/vendor/grammars/ide-tools b/vendor/grammars/ide-tools new file mode 160000 index 0000000000..2f3869e05b --- /dev/null +++ b/vendor/grammars/ide-tools @@ -0,0 +1 @@ +Subproject commit 2f3869e05b1efd0068b7eefdc0e4224035580493 diff --git a/vendor/licenses/git_submodule/ide-tools.dep.yml b/vendor/licenses/git_submodule/ide-tools.dep.yml new file mode 100644 index 0000000000..9f3b302cca --- /dev/null +++ b/vendor/licenses/git_submodule/ide-tools.dep.yml @@ -0,0 +1,31 @@ +--- +name: ide-tools +version: 2f3869e05b1efd0068b7eefdc0e4224035580493 +type: git_submodule +homepage: https://github.com/toitware/ide-tools.git +license: mit +licenses: +- sources: LICENSE + text: | + MIT License + + Copyright (c) 2021 Toit + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +notices: []