From c9f18ec69a03d6c8dfbe8714c755cd95bed23da1 Mon Sep 17 00:00:00 2001 From: Michael hughes Date: Wed, 20 Mar 2024 15:51:49 +0000 Subject: [PATCH] solrrag_coreai: SolrRag but with AI coding moving to a core subsystem --- .phpcs.xml | 105 +++++++++ course/moodleform_mod.php | 21 ++ .../plusteams/classes/data_controller.php | 48 ++++ .../plusteams/classes/field_controller.php | 0 .../lang/en/customfield_plusteams.php | 0 customfield/field/plusteams/version.php | 6 + grade/grading/form/multigraders | 1 + grade/grading/form/workflow | 1 + .../solrrag => lib}/classes/ai/aiclient.php | 0 .../classes/ai/aiexception.php | 0 .../solrrag => lib}/classes/ai/aiprovider.php | 0 .../engine/solrrag => lib}/classes/ai/api.php | 0 lib/classes/event/grade_item_hidden.php | 43 ++++ lib/classes/event/grade_item_revealed.php | 43 ++++ lib/moodlelib.php | 2 + lib/phpunit/bootstrap.php | 2 +- lib/setup.php | 3 + local/ai | 1 + local/clientevents | 1 + local/codechecker | 1 + local/plusteams | 1 + local/recompletion | 1 + mod/xaichat/lib.php | 10 +- mod/xaichat/mod_form.php | 14 +- search/engine/solrrag/classes/ai/aiclient.phx | 138 ++++++++++++ .../engine/solrrag/classes/ai/aiexception.phx | 7 + .../engine/solrrag/classes/ai/aiprovider.phx | 213 ++++++++++++++++++ search/engine/solrrag/classes/ai/api.phx | 24 ++ search/engine/solrrag/classes/engine.php | 2 +- search/engine/solrrag/lib.php | 8 +- 30 files changed, 673 insertions(+), 23 deletions(-) create mode 100644 .phpcs.xml create mode 100644 customfield/field/plusteams/classes/data_controller.php create mode 100644 customfield/field/plusteams/classes/field_controller.php create mode 100644 customfield/field/plusteams/lang/en/customfield_plusteams.php create mode 100644 customfield/field/plusteams/version.php create mode 160000 grade/grading/form/multigraders create mode 160000 grade/grading/form/workflow rename {search/engine/solrrag => lib}/classes/ai/aiclient.php (100%) rename {search/engine/solrrag => lib}/classes/ai/aiexception.php (100%) rename {search/engine/solrrag => lib}/classes/ai/aiprovider.php (100%) rename {search/engine/solrrag => lib}/classes/ai/api.php (100%) create mode 100644 lib/classes/event/grade_item_hidden.php create mode 100644 lib/classes/event/grade_item_revealed.php create mode 160000 local/ai create mode 160000 local/clientevents create mode 160000 local/codechecker create mode 160000 local/plusteams create mode 160000 local/recompletion create mode 100644 search/engine/solrrag/classes/ai/aiclient.phx create mode 100644 search/engine/solrrag/classes/ai/aiexception.phx create mode 100644 search/engine/solrrag/classes/ai/aiprovider.phx create mode 100644 search/engine/solrrag/classes/ai/api.phx diff --git a/.phpcs.xml b/.phpcs.xml new file mode 100644 index 0000000000000..7c5f2f2a8f86b --- /dev/null +++ b/.phpcs.xml @@ -0,0 +1,105 @@ + + + + node_modules/ + vendor/ + admin/tool/componentlibrary/amd/src/lunr.js + admin/tool/pluginskel/vendor/autoload.php + admin/tool/pluginskel/vendor/composer/ + admin/tool/pluginskel/vendor/symfony/ + admin/tool/pluginskel/vendor/monolog/monolog/ + admin/tool/pluginskel/vendor/psr/log/ + admin/tool/policy/amd/src/jquery-eu-cookie-law-popup.js + auth/cas/CAS/ + cache/stores/mongodb/MongoDB/ + enrol/lti/ims-blti/ + filter/algebra/AlgParser.pm + filter/tex/mimetex.* + h5p/h5plib/v124/joubel/core/ + h5p/h5plib/v124/joubel/editor/ + lib/editor/atto/plugins/html/yui/src/codemirror/ + lib/editor/atto/plugins/html/yui/src/beautify/ + lib/editor/atto/yui/src/rangy/js/*.* + lib/editor/tiny/js/tinymce/ + lib/editor/tinymce/plugins/pdw/tinymce/ + lib/editor/tinymce/plugins/spellchecker/rpc.php + lib/editor/tinymce/tiny_mce/ + lib/mlbackend/php/phpml/ + lib/adodb/ + lib/behat/axe/ + lib/bennu/ + lib/evalmath/ + lib/phpspreadsheet/ + lib/google/ + lib/htmlpurifier/ + lib/minify/matthiasmullie-minify/ + lib/minify/matthiasmullie-pathconverter/ + lib/pear/HTML/Common.php + lib/pear/HTML/QuickForm.php + lib/pear/HTML/QuickForm/ + lib/pear/PEAR.php + lib/phpmailer/ + lib/simplepie/ + lib/tcpdf/ + lib/yuilib/ + lib/yuilib/gallery/ + lib/jquery/ + lib/html2text/ + lib/markdown/ + lib/xhprof/ + lib/horde/ + lib/requirejs/ + lib/amd/src/loglevel.js + lib/mustache/ + lib/amd/src/mustache.js + lib/graphlib.php + lib/php-css-parser/ + lib/rtlcss/ + lib/scssphp/ + lib/spout/ + lib/amd/src/chartjs-lazy.js + lib/maxmind/GeoIp2/ + lib/maxmind/MaxMind/ + lib/ltiprovider/ + lib/lti1p3/ + lib/amd/src/truncate.js + lib/fonts/ + lib/amd/src/adapter.js + lib/validateurlsyntax.php + lib/amd/src/popper.js + lib/geopattern-php/ + lib/php-jwt/ + lib/polyfills/ + lib/emoji-data/ + lib/plist/ + lib/zipstream/ + lib/php-enum/ + lib/http-message/ + lib/phpxmlrpc/ + local/codechecker/phpcs/ + local/codechecker/PHPCompatibility/ + media/player/videojs/amd/src/video-lazy.js + media/player/videojs/amd/src/Youtube-lazy.js + media/player/videojs/videojs/ + media/player/videojs/amd/src/local/ogv/ogv.js + media/player/videojs/ogvjs/ + media/player/videojs/amd/src/videojs-ogvjs-lazy.js + mod/assign/feedback/editpdf/fpdi/ + repository/s3/S3.php + theme/boost/scss/bootstrap/ + theme/boost/amd/src/bootstrap/alert.js + theme/boost/amd/src/bootstrap/button.js + theme/boost/amd/src/bootstrap/carousel.js + theme/boost/amd/src/bootstrap/collapse.js + theme/boost/amd/src/bootstrap/dropdown.js + theme/boost/amd/src/bootstrap/modal.js + theme/boost/amd/src/bootstrap/popover.js + theme/boost/amd/src/bootstrap/tools/sanitizer.js + theme/boost/amd/src/bootstrap/scrollspy.js + theme/boost/amd/src/bootstrap/tab.js + theme/boost/amd/src/bootstrap/toast.js + theme/boost/amd/src/bootstrap/tooltip.js + theme/boost/amd/src/bootstrap/util.js + theme/boost/amd/src/index.js + theme/boost/scss/fontawesome/ + diff --git a/course/moodleform_mod.php b/course/moodleform_mod.php index 3357140484a15..e593eaec2b5b6 100644 --- a/course/moodleform_mod.php +++ b/course/moodleform_mod.php @@ -1195,4 +1195,25 @@ public function get_data() { } return $data; } + + public function standard_aiprovider_coursemodule_elements() { + + if (!$this->_features->ai) { + return; + } + $mform = $this->_form; + // Adding the rest of mod_xaichat settings, spreading all them into this fieldset + // ... or adding more fieldsets ('header' elements) if needed for better logic. + $mform->addElement('header', 'aiprovider', get_string('aiprovider')); + + $providers = \core\ai\api::get_all_providers(); + $optproviders = []; + foreach($providers as $provider) { + $optproviders[$provider->get('id')] = $provider->get('name'); + } + $mform->addElement('select', 'aiproviderid', + 'Choose Provider', + $optproviders + ); + } } diff --git a/customfield/field/plusteams/classes/data_controller.php b/customfield/field/plusteams/classes/data_controller.php new file mode 100644 index 0000000000000..d39514b5a7bec --- /dev/null +++ b/customfield/field/plusteams/classes/data_controller.php @@ -0,0 +1,48 @@ +get_field()->get_configdata_property('defaultvalue'); + if ('' . $defaultvalue !== '') { + $key = array_search($defaultvalue, $this->get_field()->get_options()); + if ($key !== false) { + return $key; + } + } + return 0; + } + + public function instance_form_definition(\MoodleQuickForm $mform) { +// $field = $this->get_field(); +// $config = $field->get('configdata'); +// $options = $field->get_options(); +// $formattedoptions = array(); +// $context = $this->get_field()->get_handler()->get_configuration_context(); +// local_plusteams_linkform($mform); + } + public function instance_form_validation(array $data, array $files) : array { + + } + public function export_value() { + $value = $this->get_value(); + + if ($this->is_empty($value)) { + return null; + } + + $options = $this->get_field()->get_options(); + if (array_key_exists($value, $options)) { + return format_string($options[$value], true, + ['context' => $this->get_field()->get_handler()->get_configuration_context()]); + } + + return null; + } +} \ No newline at end of file diff --git a/customfield/field/plusteams/classes/field_controller.php b/customfield/field/plusteams/classes/field_controller.php new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/customfield/field/plusteams/lang/en/customfield_plusteams.php b/customfield/field/plusteams/lang/en/customfield_plusteams.php new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/customfield/field/plusteams/version.php b/customfield/field/plusteams/version.php new file mode 100644 index 0000000000000..f4552b133d4c9 --- /dev/null +++ b/customfield/field/plusteams/version.php @@ -0,0 +1,6 @@ +component = 'customfield_plusteams'; +$plugin->version = 2022041900; +$plugin->requires = 2022041200; diff --git a/grade/grading/form/multigraders b/grade/grading/form/multigraders new file mode 160000 index 0000000000000..aec0ab5d4aa5d --- /dev/null +++ b/grade/grading/form/multigraders @@ -0,0 +1 @@ +Subproject commit aec0ab5d4aa5dce02bdf9336784ebe14aa10b9e3 diff --git a/grade/grading/form/workflow b/grade/grading/form/workflow new file mode 160000 index 0000000000000..eb0a4bd5c7587 --- /dev/null +++ b/grade/grading/form/workflow @@ -0,0 +1 @@ +Subproject commit eb0a4bd5c75872357e85934f1eaa1035bfa12cd0 diff --git a/search/engine/solrrag/classes/ai/aiclient.php b/lib/classes/ai/aiclient.php similarity index 100% rename from search/engine/solrrag/classes/ai/aiclient.php rename to lib/classes/ai/aiclient.php diff --git a/search/engine/solrrag/classes/ai/aiexception.php b/lib/classes/ai/aiexception.php similarity index 100% rename from search/engine/solrrag/classes/ai/aiexception.php rename to lib/classes/ai/aiexception.php diff --git a/search/engine/solrrag/classes/ai/aiprovider.php b/lib/classes/ai/aiprovider.php similarity index 100% rename from search/engine/solrrag/classes/ai/aiprovider.php rename to lib/classes/ai/aiprovider.php diff --git a/search/engine/solrrag/classes/ai/api.php b/lib/classes/ai/api.php similarity index 100% rename from search/engine/solrrag/classes/ai/api.php rename to lib/classes/ai/api.php diff --git a/lib/classes/event/grade_item_hidden.php b/lib/classes/event/grade_item_hidden.php new file mode 100644 index 0000000000000..443a422107f11 --- /dev/null +++ b/lib/classes/event/grade_item_hidden.php @@ -0,0 +1,43 @@ +. + +/** + * Grade item updated event. + * + * @package core + * @copyright 2019 Dmitrii Metelkin + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace core\event; + +defined('MOODLE_INTERNAL') || die(); + +class grade_item_hidden extends grade_item_updated { + protected function init() { + $this->data['objecttable'] = 'grade_items'; + $this->data['crud'] = 'u'; + $this->data['edulevel'] = self::LEVEL_OTHER; + } + public static function get_name() { + return get_string('eventgradeitemhidden', 'core_grades'); + } + public function get_description() { + return "The user with id '" . $this->userid . "' hid a grade item with id '" . $this->objectid . "'" . + " of type '" . $this->other['itemtype'] . "' and name '" . $this->other['itemname'] . "'" . + " in the course with the id '" . $this->courseid . "'."; + } +} diff --git a/lib/classes/event/grade_item_revealed.php b/lib/classes/event/grade_item_revealed.php new file mode 100644 index 0000000000000..1353fda69b2d0 --- /dev/null +++ b/lib/classes/event/grade_item_revealed.php @@ -0,0 +1,43 @@ +. + +/** + * Grade item updated event. + * + * @package core + * @copyright 2019 Dmitrii Metelkin + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace core\event; + +defined('MOODLE_INTERNAL') || die(); + +class grade_item_revealed extends grade_item_updated { + protected function init() { + $this->data['objecttable'] = 'grade_items'; + $this->data['crud'] = 'u'; + $this->data['edulevel'] = self::LEVEL_OTHER; + } + public static function get_name() { + return get_string('eventgradeitemrevealed', 'core_grades'); + } + public function get_description() { + return "The user with id '" . $this->userid . "' revealed a grade item with id '" . $this->objectid . "'" . + " of type '" . $this->other['itemtype'] . "' and name '" . $this->other['itemname'] . "'" . + " in the course with the id '" . $this->courseid . "'."; + } +} diff --git a/lib/moodlelib.php b/lib/moodlelib.php index 79053882ceda3..049c8574daa0f 100644 --- a/lib/moodlelib.php +++ b/lib/moodlelib.php @@ -421,6 +421,8 @@ define('FEATURE_CONTROLS_GRADE_VISIBILITY', 'controlsgradevisbility'); /** True if module supports plagiarism plugins */ define('FEATURE_PLAGIARISM', 'plagiarism'); +/** True if module uses ai provider. */ +define('FEATURE_AI', 'ai'); /** True if module has code to track whether somebody viewed it */ define('FEATURE_COMPLETION_TRACKS_VIEWS', 'completion_tracks_views'); diff --git a/lib/phpunit/bootstrap.php b/lib/phpunit/bootstrap.php index 0ac815e3143ff..f7ca43a4c0ef2 100644 --- a/lib/phpunit/bootstrap.php +++ b/lib/phpunit/bootstrap.php @@ -251,10 +251,10 @@ $CFG->profilingenabled = true; $CFG->profilingincluded = '*'; } + require("$CFG->dirroot/lib/setup.php"); raise_memory_limit(MEMORY_HUGE); - if (PHPUNIT_UTIL) { // We are not going to do testing, this is 'true' in utility scripts that only init database. return; diff --git a/lib/setup.php b/lib/setup.php index cba63271ab666..c8af3b8f568cb 100644 --- a/lib/setup.php +++ b/lib/setup.php @@ -653,6 +653,7 @@ setup_DB(); if (PHPUNIT_TEST and !PHPUNIT_UTIL) { + // Make sure tests do not run in parallel. $suffix = ''; if (phpunit_util::is_in_isolated_process()) { @@ -663,7 +664,9 @@ try { if ($dbhash = $DB->get_field('config', 'value', array('name'=>'phpunittest'))) { // reset DB tables + echo('1'); phpunit_util::reset_database(); + echo('2'); } } catch (Exception $e) { if ($dbhash) { diff --git a/local/ai b/local/ai new file mode 160000 index 0000000000000..a41b8271f47db --- /dev/null +++ b/local/ai @@ -0,0 +1 @@ +Subproject commit a41b8271f47dbadfe33a95607c87acf5d0d6c588 diff --git a/local/clientevents b/local/clientevents new file mode 160000 index 0000000000000..24ed0b7798607 --- /dev/null +++ b/local/clientevents @@ -0,0 +1 @@ +Subproject commit 24ed0b779860770add6d51ca226fd2e943e2508f diff --git a/local/codechecker b/local/codechecker new file mode 160000 index 0000000000000..9708c27f3a249 --- /dev/null +++ b/local/codechecker @@ -0,0 +1 @@ +Subproject commit 9708c27f3a24933863c2ca66a0ecbfcfb6cc80b9 diff --git a/local/plusteams b/local/plusteams new file mode 160000 index 0000000000000..3e0a109ed6eb1 --- /dev/null +++ b/local/plusteams @@ -0,0 +1 @@ +Subproject commit 3e0a109ed6eb10770a230dbe9abb12ad90a10873 diff --git a/local/recompletion b/local/recompletion new file mode 160000 index 0000000000000..2ed9fc6d3d853 --- /dev/null +++ b/local/recompletion @@ -0,0 +1 @@ +Subproject commit 2ed9fc6d3d853cece1bc92293294457f8ac890dc diff --git a/mod/xaichat/lib.php b/mod/xaichat/lib.php index 6c7e078f64d52..d3e0fdf4cb6f6 100644 --- a/mod/xaichat/lib.php +++ b/mod/xaichat/lib.php @@ -23,10 +23,10 @@ */ defined('MOODLE_INTERNAL') || die(); - -require_once($CFG->dirroot ."/search/engine/solrrag/classes/ai/api.php"); -require_once($CFG->dirroot ."/search/engine/solrrag/classes/ai/aiprovider.php"); -require_once($CFG->dirroot ."/search/engine/solrrag/classes/ai/aiclient.php"); +// +//require_once($CFG->dirroot ."/search/engine/solrrag/classes/ai/api.php"); +//require_once($CFG->dirroot ."/search/engine/solrrag/classes/ai/aiprovider.php"); +//require_once($CFG->dirroot ."/search/engine/solrrag/classes/ai/aiclient.php"); /** * Return if the plugin supports $feature. @@ -36,6 +36,8 @@ */ function xaichat_supports($feature) { switch ($feature) { + case FEATURE_AI: + return true; case FEATURE_GRADE_HAS_GRADE: return true; case FEATURE_MOD_INTRO: diff --git a/mod/xaichat/mod_form.php b/mod/xaichat/mod_form.php index 1b88e0a319eac..4de0909adf935 100644 --- a/mod/xaichat/mod_form.php +++ b/mod/xaichat/mod_form.php @@ -69,19 +69,7 @@ public function definition() { $this->add_intro_editor(); } - // Adding the rest of mod_xaichat settings, spreading all them into this fieldset - // ... or adding more fieldsets ('header' elements) if needed for better logic. - $mform->addElement('header', 'xaichatfieldset', get_string('xaichatfieldset', 'mod_xaichat')); - - $providers = \core\ai\api::get_all_providers(); - $optproviders = []; - foreach($providers as $provider) { - $optproviders[$provider->get('id')] = $provider->get('name'); - } - $mform->addElement('select', 'aiproviderid', - 'Choose Provider', - $optproviders - ); + $this->standard_aiprovider_coursemodule_elements(); // Add standard grading elements. $this->standard_grading_coursemodule_elements(); diff --git a/search/engine/solrrag/classes/ai/aiclient.phx b/search/engine/solrrag/classes/ai/aiclient.phx new file mode 100644 index 0000000000000..086c8d3a92645 --- /dev/null +++ b/search/engine/solrrag/classes/ai/aiclient.phx @@ -0,0 +1,138 @@ +libdir.'/filelib.php'); +use core\ai\AiException; +/** + * Base client for AI providers that uses simple http request. + */ +class AIClient extends \curl { + /** + * @var AIProvider + */ + private $provider; + public function __construct( + \core\ai\AIProvider $provider + ) { + $this->provider = $provider; + $settings = []; + parent::__construct($settings); + $this->setHeader('Authorization: Bearer ' . $this->provider->get('apikey')); + $this->setHeader('Content-Type: application/json'); + } + + public function get_embeddings_url(): string { + return $this->provider->get('baseurl') . $this->provider->get('embeddings'); + } + + public function get_chat_completions_url(): string { + return $this->provider->get('baseurl') . $this->provider->get('completions'); + } + + /** + * @param $messages + * @return array String array of each line of the AI's Response. + * @throws \coding_exception + */ + public function chat($messages) { + $params = [ + "model" => $this->provider->get('completionmodel'), + "messages" => $messages + ]; + $params = json_encode($params); + $rawresult = $this->post($this->get_chat_completions_url(), $params); + $jsonresult = json_decode($rawresult); + if (isset($jsonresult->error)) { + throw new AiException("Error: " . $jsonresult->error->message . ":". print_r($messages, true)); + //return "Error: " . $jsonresult->error->message . ":". print_r($messages, true); + } + $result = []; + if (isset($jsonresult->choices)) { + $result = $this->convert_chat_completion($jsonresult->choices); + if (isset($jsonresult->usage)) { + $this->provider->increment_prompt_usage($jsonresult->usage->prompt_tokens); + $this->provider->increment_completion_tokens($jsonresult->usage->completion_tokens); + $this->provider->increment_total_tokens($jsonresult->usage->total_tokens); + } + } + + return $result; + } + + /** + * Converts an OpenAI Type of response to an array of sentences + * @param $completion + * @return array + */ + protected function convert_chat_completion($choices) { + $responses = []; + foreach($choices as $choice) { + array_push($responses, $choice->message); + } + return $responses; + } + /** + * @param $document + * @return array + */ + public function embed_query($content): array { + // Send document to back end and return the vector + $usedptokens = $this->provider->get_usage('prompt_tokens'); + $totaltokens = $this->provider->get_usage('total_tokens'); + // mtrace("Prompt tokens: $usedptokens. Total tokens: $totaltokens"); + $params = [ + "input" => htmlentities($content), // TODO need to do some length checking here! + "model" => $this->provider->get('embeddingmodel') + ]; + $params = json_encode($params); +// var_dump($this->get_embeddings_url()); + + $rawresult = $this->post($this->get_embeddings_url(), $params); +// var_dump($rawresult); + $result = json_decode($rawresult, true); + // var_dump($result); + $usage = $result['usage']; + $this->provider->increment_prompt_usage($usage['prompt_tokens']); + $this->provider->increment_total_tokens($usage['total_tokens']); + // mtrace("Used Prompt tokens: {$usage['prompt_tokens']}. Total tokens: {$usage['total_tokens']}"); + $data = $result['data']; + foreach($data as $d) { + if ($d['object'] == "embedding") { + return $d['embedding']; + } + } + $usedptokens = $this->provider->get_usage('prompt_tokens'); + $totaltokens = $this->provider->get_usage('total_tokens'); + // mtrace("Total Used: Prompt tokens: $usedptokens. Total tokens: $totaltokens"); + return []; + } + public function embed_documents(array $documents) { + // Go send the documents off to a back end and then return array of each document's vectors. + // But for the minute generate an array of fake vectors of a specific length. + $embeddings = []; + foreach($documents as $doc) { + $embeddings[] = $this->embed_query($doc); + } + return $embeddings; + } + public function fake_embed(array $documents) { + $vectors = []; + foreach ($documents as $document) { + $vectors[] = $this->fake_vector(1356); + } + return $vectors; + } + public function complete($query) { + + + } + private function fake_vector($length) { + $vector = []; + for ($i = 0; $i < $length; $i++) { + $vector[] = rand(0, 1); + } + return $vector; + } + + + +} diff --git a/search/engine/solrrag/classes/ai/aiexception.phx b/search/engine/solrrag/classes/ai/aiexception.phx new file mode 100644 index 0000000000000..ab006c952e12b --- /dev/null +++ b/search/engine/solrrag/classes/ai/aiexception.phx @@ -0,0 +1,7 @@ +dirroot . "/search/engine/solrrag/lib.php"); + +use \core\persistent; +use core_course_category; + +class AIProvider extends persistent { +// Ultimately this would extend a persistent. + const CONTEXT_ALL_MY_COURSES = -1; + + protected static function define_properties() + { + return [ + 'name' => [ + 'type' => PARAM_TEXT + ], + 'apikey' =>[ + 'type' => PARAM_ALPHANUMEXT + ], + 'allowembeddings' => [ + 'type' => PARAM_BOOL + ], + 'allowquery' => [ + 'type' => PARAM_BOOL + ], + 'baseurl' => [ + 'type' => PARAM_URL + ], + 'embeddings' => [ + 'type' => PARAM_URL + ], + 'embeddingmodel' => [ + 'type' => PARAM_ALPHANUMEXT + ], + 'completions' => [ + 'type' => PARAM_URL + ], + 'completionmodel' => [ + 'type' => PARAM_ALPHANUMEXT + ], + // What context is this provider attached to. + // If null, it's a global provider. + // If -1 its limited to user's own courses. + 'context' => [ + 'type' => PARAM_INT + ], + // If true, only courses that the user is enrolled in will be searched. + 'onlyenrolledcourses' => [ + 'type' => PARAM_BOOL + ], + ]; + } + + public function use_for_embeddings(): bool { + return $this->get('allowembeddings'); + } + + public function use_for_query():bool { + return $this->get('allowquery'); + } + public function get_usage($type) { + return "-"; + $key = [ + '$type', + $this->get('id'), + $this->get('apikey'), + ]; + $current = get_config('ai', $key); + return $current; + } + public function increment_prompt_usage($change) { + return; + $key = [ + 'prompttokens', + $this->get('id'), + $this->get('apikey'), + ]; + $key = implode("_", $key); + $current = get_config('ai', $key); + $new = $current + $change; + set_config($key, $new, 'ai'); + } + public function increment_completion_tokens($change) { + return; + $key = [ + 'completiontokens', + $this->get('id'), + $this->get('apikey'), + ]; + $key = implode("_", $key); + $current = get_config('ai', $key); + $new = $current + $change; + set_config($key, $new, 'ai'); + } + public function increment_total_tokens($change) { + return; + $key = [ + 'totaltokens', + $this->get('id'), + $this->get('apikey'), + ]; + $key = implode("_", $key); + $current = get_config('ai', $key); + $new = $current + $change; + set_config($key, $new, 'ai'); + } + + /** + * Returns appropriate search settings based on + * provider configuration. + */ + public function get_settings() { + // `userquery` and `vector` will be filled at run time. + $settings = [ + 'userquery'=> null, + 'vector' => null, + // `similarity` is a boolean that determines if the query should use vector similarity search. + 'similarity' => true, + 'areaids' => [], + // `excludeareaids` is an array of areaids that should be excluded from the search. + 'excludeareaids'=> ["core_user-user"], // <-- This may be should be in control of the AI Provider. + 'courseids' => [], // This of course IDs that result should be limited to. + ]; + return $settings; + } + + /** + * Gets user specific settings. + * + * This takes on some of the function that the manager code did. + */ + public function get_settings_for_user($user) { + $usersettings = $this->get_settings(); + + // This is basically manager::build_limitcourseids(). + $mycourseids = enrol_get_my_courses(array('id', 'cacherev'), 'id', 0, [], false); + $onlyenrolledcourses = $this->get('onlyenrolledcourses'); + $courseids = []; + if ($this->get('context') == self::CONTEXT_ALL_MY_COURSES) { + $courseids = array_keys($mycourseids); + } else { + $context = \context::instance_by_id($this->get('context')); + if ($context->contextlevel == CONTEXT_COURSE) { + // Check that the specific course is also in the user's list of courses. + $courseids = array_intersect([$context->instanceid], $mycourseids); + } else if ($context->contextlevel == CONTEXT_COURSECAT) { + // CourseIDs will be all courses in the category, + // optionally that the user is enrolled in + $category = core_course_category::get($context->instanceid); + $categorycourseids = $category->get_courses([ + 'recursive'=>true, + 'idonly' => true + ]); + } else if ($context->contextlevel == CONTEXT_SYSTEM) { + // No restrictions anywhere. + } + } + $usersettings['courseids'] = $courseids; + + return $usersettings; + } + + //public function + // TODO token counting. + /** + * We're overriding this whilst we don't have a real DB table. + * @param $filters + * @param $sort + * @param $order + * @param $skip + * @param $limit + * @return array + */ + public static function get_records($filters = array(), $sort = '', $order = 'ASC', $skip = 0, $limit = 0) { + global $_ENV; + $records = []; + $fake = new static(0, (object) [ + 'id' => 1, + 'name' => "Fake Open AI Provider", + 'allowembeddings' => true, + 'allowquery' => true, + 'baseurl' => 'https://api.openai.com/v1/', + 'embeddings' => 'embeddings', + 'embeddingmodel' => 'text-embedding-3-small', + 'completions' => 'chat/completions', + 'completionmodel' => 'gpt-4-turbo-preview', + 'apikey'=> $_ENV['OPENAIKEY'], + 'context' => \context_system::instance()->id, + //null, // Global AI Provider + 'onlyenrolledcourses' => true + ]); + array_push($records, $fake); + $fake = new static(0, (object) [ + 'id' => 2, + 'name' => "Ollama AI Provider", + 'allowembeddings' => true, + 'allowquery' => true, + 'baseurl' => 'http://127.0.0.1:11434/api/', + 'embeddings' => 'embeddings', + 'embeddingmodel' => '', + 'completions' => 'chat', + 'completionmodel' => 'llama2', + 'context' => null, // Global AI Provider + 'onlyenrolledcourses' => true + // 'apikey'=> $_ENV['OPENAIKEY'] + ]); + array_push($records, $fake); + return $records; + } + +} diff --git a/search/engine/solrrag/classes/ai/api.phx b/search/engine/solrrag/classes/ai/api.phx new file mode 100644 index 0000000000000..be8c4d627b68c --- /dev/null +++ b/search/engine/solrrag/classes/ai/api.phx @@ -0,0 +1,24 @@ +dirroot . "/search/engine/solrrag/lib.php"); + +class api { + + /** + * Return a list of AIProviders that are available for specified context. + * @param $context + * @return array + */ + public static function get_all_providers($context = null) { + return array_values(AIProvider::get_records()); + } + public static function get_provider(int $id): AIProvider { + $fakes = AIProvider::get_records(); + return $fakes[0]; // Open AI + // return $fakes[1]; // Ollama + + } +} diff --git a/search/engine/solrrag/classes/engine.php b/search/engine/solrrag/classes/engine.php index 10a5cbcbf2405..043102731605b 100644 --- a/search/engine/solrrag/classes/engine.php +++ b/search/engine/solrrag/classes/engine.php @@ -4,7 +4,7 @@ use search_solrrag\document; use search_solrrag\schema; -require_once($CFG->dirroot . "/search/engine/solrrag/lib.php"); +//require_once($CFG->dirroot . "/search/engine/solrrag/lib.php"); // // Fudge autoloading! // require_once($CFG->dirroot ."/search/engine/solrrag/classes/ai/api.php"); // require_once($CFG->dirroot ."/search/engine/solrrag/classes/ai/aiprovider.php"); diff --git a/search/engine/solrrag/lib.php b/search/engine/solrrag/lib.php index 46f51c2d8e667..128c498db2389 100644 --- a/search/engine/solrrag/lib.php +++ b/search/engine/solrrag/lib.php @@ -1,6 +1,6 @@ dirroot ."/search/engine/solrrag/classes/ai/api.php"); -require_once($CFG->dirroot ."/search/engine/solrrag/classes/ai/aiprovider.php"); -require_once($CFG->dirroot ."/search/engine/solrrag/classes/ai/aiclient.php"); -require_once($CFG->dirroot ."/search/engine/solrrag/classes/ai/aiexception.php"); \ No newline at end of file +//require_once($CFG->dirroot ."/search/engine/solrrag/classes/ai/api.php"); +//require_once($CFG->dirroot ."/search/engine/solrrag/classes/ai/aiprovider.php"); +//require_once($CFG->dirroot ."/search/engine/solrrag/classes/ai/aiclient.php"); +//require_once($CFG->dirroot ."/search/engine/solrrag/classes/ai/aiexception.php");