Skip to content

Commit

Permalink
Per-user course constraint on rag and CLI to test RAG
Browse files Browse the repository at this point in the history
  • Loading branch information
mhughes2k committed Apr 22, 2024
1 parent 1590ebb commit 32728fe
Show file tree
Hide file tree
Showing 11 changed files with 197 additions and 26 deletions.
4 changes: 2 additions & 2 deletions ai/classes/aiclient.php
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,11 @@ public function __construct(
}

public function get_embeddings_url(): string {
return $this->provider->get('baseurl') . $this->provider->get('embeddings');
return $this->provider->get('baseurl') . $this->provider->get('embeddingsurl');
}

public function get_chat_completions_url(): string {
return $this->provider->get('baseurl') . $this->provider->get('completions');
return $this->provider->get('baseurl') . $this->provider->get('completionsurl');
}

/**
Expand Down
19 changes: 14 additions & 5 deletions ai/classes/aiprovider.php
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,10 @@ public function get_settings_for_user($user) {
$courseids = [];
if ($this->get('contextid') == self::CONTEXT_ALL_MY_COURSES) {
$courseids = array_keys($mycourseids);
} else if ($this->get('contextid') == 0) {
$context = \context_system::instance();
} else {

$context = \context::instance_by_id($this->get('contextid'));
if ($context->contextlevel == CONTEXT_COURSE) {
// Check that the specific course is also in the user's list of courses.
Expand All @@ -179,7 +182,7 @@ public function get_settings_for_user($user) {
// TODO token counting.
/**
* We're overriding this whilst we don't have a real DB table.
* @param $filters
* @param $filters This can have the contexts, but also can have 'excludesystem' set to not include system AI Providers.
* @param $sort
* @param $order
* @param $skip
Expand All @@ -188,7 +191,6 @@ public function get_settings_for_user($user) {
*/
public static function get_records($filters = [], $sort = '', $order = 'ASC', $skip = 0, $limit = 0) {
global $_ENV;
$records = parent::get_records($filters, $sort, $order, $skip, $limit);
// $records = [];
// $fake = new static(0, (object) [
// 'id' => 1,
Expand Down Expand Up @@ -247,6 +249,13 @@ public static function get_records($filters = [], $sort = '', $order = 'ASC', $s
} else {
$targetcontext = \context::instance_by_id($targetcontextid);
}
$systemproviders = [];
if (!isset($filters['excludesystem'])) {
$systemfilter = $filters;
$systemfilter['contextid'] = 0;
$systemproviders = parent::get_records($systemfilter, $sort, $order, $skip, $limit);
}
$records = parent::get_records($filters, $sort, $order, $skip, $limit);
$records = array_filter($records, function($record) use ($filters, $targetcontext) {
$result = true;
foreach($filters as $key => $value) {
Expand All @@ -255,15 +264,15 @@ public static function get_records($filters = [], $sort = '', $order = 'ASC', $s
if ($providercontextid == self::CONTEXT_ALL_MY_COURSES) {
// More problematic.
$result = $result & true;
} else if ($providercontextid == null) {
} else if ($providercontextid == 0) {
// System provider so always matches.
$result = $result & true;
} else {
$providercontext = \context::instance_by_id(
$providercontextid
);
$ischild = $targetcontext->is_child_of($providercontext, true);
debugging("IS child ". (int)$ischild, DEBUG_DEVELOPER);
// debugging("IS child ". (int)$ischild, DEBUG_DEVELOPER);
$result = $result & $ischild;
}
}else {
Expand All @@ -275,7 +284,7 @@ public static function get_records($filters = [], $sort = '', $order = 'ASC', $s
}
return $result;
});

$records = array_merge($systemproviders, $records);
return $records;
}

Expand Down
5 changes: 3 additions & 2 deletions ai/classes/api.php
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,13 @@ public static function get_provider(int $id): AIProvider {
* @return array
*/
public static function get_providers($contextid = null, $allowchat = null, $allowembeddings = null) {
$requirements = ['contextid','allowchat', 'allowembeddings'];
$requirements = ['contextid', 'allowchat', 'allowembeddings'];
// Filtering AI providers that are available to $contextid, walking up the
// tree when we only have the contextid the AIProvider is set *on* is going to take
// more work.
$filters = [];
foreach($requirements as $req) {

$reqparam = ${$req};
// Null means we don't consider it.
if (!is_null($reqparam)) {
Expand All @@ -50,7 +51,7 @@ public static function get_providers($contextid = null, $allowchat = null, $allo
$filters[$req] = $reqparam;
}
}
//debugging(print_r($filters,1), DEBUG_DEVELOPER);
debugging(print_r($filters,1), DEBUG_DEVELOPER);
$providers = aiprovider::get_records($filters);
return array_values($providers);
}
Expand Down
5 changes: 3 additions & 2 deletions ai/classes/form/openaiapiprovider.php
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,16 @@ public function definition() {

$mform = $this->_form;
$provider = $this->get_persistent();
$mform->addElement('html','intro', 'hello');

// $mform->addElement('html','intro', 'hello');
$mform->addElement('header', 'features', get_string('general', 'ai'));
// Name.
$mform->addElement('text', 'name', get_string('providername', 'ai'));
$mform->addRule('name', null, 'required', null, 'client');
$mform->addRule('name', get_string('maximumchars', '', 255), 'maxlength', 255, 'client');
$mform->addHelpButton('name', 'providername', 'ai');
$mform->addElement('advcheckbox', 'enabled', get_string('enabled', 'ai'));
$mform->addHelpButton('enabled', 'enabled', 'ai');
$mform->setDefault('enabled', true);
// Client Secret.
$mform->addElement('text','baseurl', get_string('baseurl', 'ai'));
$mform->setType('baseurl', PARAM_URL);
Expand Down
19 changes: 19 additions & 0 deletions ai/classes/logger.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
<?php
namespace core_ai;

class logger {
protected $logpath;
function __construct($identifier) {
$logdir = make_temp_directory('ai', true);
$this->logpath = $logdir . '/' . $identifier . '.log';
}
public function write($message) {
$ts = microtime(true);
$f = fopen($this->logpath, 'a');
if(flock($f, LOCK_EX | LOCK_NB)) {
fwrite($f, "{$ts} - {$message}\n");
flock($f, LOCK_UN);
}
fclose($f);
}
}
104 changes: 104 additions & 0 deletions ai/cli/query.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
<?php
define('CLI_SCRIPT', true);

require_once(__DIR__.'/../../config.php');
require_once($CFG->libdir.'/clilib.php');
use core_ai\api;
use core_ai\aiclient;
use core_ai\aiprovider;
[$options, $unrecognized] = cli_get_params(
[
'help' => false,
'prompt' => false,
'providerid' => 1,
'courses' => [2],
'contexts' => [],
'limit' => 3,
'userid' => false
],[
'h' => 'help',
'p' => 'providerid',
'l' => 'limit'
]

);

if ($unrecognized) {
$unrecognized = implode("\n ", $unrecognized);
cli_error(get_string('cliunknowoption', 'admin', $unrecognized));
}
$help = <<<EOT
Ad hoc cron tasks.
Options:
-h, --help Print out this help
-p, --providerid ID of provider to use
--courses IDs of courses to use
--userid ID of user to search as
Run all queued tasks:
\$sudo -u www-data /usr/bin/php admin/cli/adhoc_task.php --execute
EOT;
if ($options['help']) {
echo $help;
exit(0);
}

var_dump($options);
var_dump($unrecognized);

$providerid = $options['providerid'];
$filters = [];
$accessinfo = [];
$limit = $options['limit'];

if (empty($options['prompt'])) {
throw new \moodle_exception("Prompt is required");
}

execute($options);
function execute( $options) {
global $USER;
$humantimenow = date('r', time());
mtrace("Server Time: {$humantimenow}\n");
$user = $USER;
if (!empty($options['userid'])) {
$user = \core_user::get_user($options['userid']);
}
// var_dump($user);
mtrace("Searching as ". fullname($user));
\core\session\manager::init_empty_session();
\core\session\manager::set_user($user);
$providerid = $options['providerid'];
$provider = api::get_provider($providerid);
$client = new aiclient($provider);

$manager = \core_search\manager::instance();
$formdata = new \stdClass();
$formdata->userprompt = $options['prompt'];
$formdata->contextids = [];
$formdata->mycoursesonly = false;
$formdata->courseids = $options['courses'];

$settings = $provider->get_settings_for_user($user);
$settings['userquery'] = $formdata->userprompt;
$settings['courseids'] = $options['courses'];
var_dump($settings);
$vector = $client->embed_query($formdata->userprompt);
$settings['vector'] = $vector;

$limitcourseids = $manager->build_limitcourseids($formdata);
$limitcontextids = $formdata->contextids;
$contextinfo = $manager->get_areas_user_accesses($limitcourseids, $limitcontextids);

mtrace("Manager settings");
mtrace("Restrict to courses: ");
var_dump($limitcourseids);
mtrace("Contextinfo returned: ");
var_dump($contextinfo);

mtrace('Performing search');
$docs = $manager->search((object)$settings);
var_dump($docs);

}
25 changes: 15 additions & 10 deletions ai/index.php
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@
$mform = new \core_ai\form\openaiapiprovider(null, [
'persistent' => $provider,
'type' => $type,
'contextid' => $incontextid
'contextid' => $incontextid,
'enabled' => true,
]);
}

Expand All @@ -64,20 +65,24 @@
} else if ($action == api::ACTION_EDIT_PROVIDER) {
// Handle edit.
if ($mform->is_cancelled()) {
echo 'cancelled';
redirect(new moodle_url('/ai/index.php'));
}
if ($mform->is_submitted()) {
echo 'submitted';
}
echo 'validated '. (int)$mform->is_validated();

if ($data = $mform->get_data()) {
var_dump($data);
if (!empty($data->id)) {
core_ai\api::update_provider($data);
} else {
core_ai\api::create_provider($data);
try {
if (!empty($data->id)) {
core_ai\api::update_provider($data);
} else {
core_ai\api::create_provider($data);
}
redirect(new moodle_url('/ai/index.php'));
}
catch (moodle_exception $e) {
throw $e;
}


exit();
} else {
echo $OUTPUT->header();
Expand Down
19 changes: 19 additions & 0 deletions ai/test.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
<?php

/**
* A lot of this is based on the Oauth2 issuers.php file.
*/
require_once("../config.php");

if (!debugging("Debug Mode", DEBUG_DEVELOPER)) {
throw new moodle_exception('notpermitted');
exit();
}
echo \html_writer::start_tag("pre");
use core_ai\api;
$providers = api::get_providers(11);

var_dump($providers);


echo \html_writer::end_tag("pre");
6 changes: 6 additions & 0 deletions lang/en/ai.php
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,11 @@
$string['addprovider'] = 'Add AI Provider';
$string['anyusercourse'] = 'Any course user is enrolled in';
$string['anywhere'] = 'Anywhere in site';
$string['baseurl_help'] = 'Common shared base URL *without* trailing slash';
$string['enabled'] = 'Enabled';
$string['enabled_help'] = 'Show this AI Provider be available to users.';
$string['disabled'] = 'Disabled';
$string['general'] = 'General Settings';
$string['removeprovider'] = 'Remove AI Provider';
$string['manageproviders'] = 'Manage AI Providers';
$string['availableproviders'] = 'Available Providers';
Expand All @@ -44,6 +47,9 @@
$string['disable'] = 'Disable';
$string['embedding'] = 'Embedding';
$string['embedding_help'] = 'Embedding allows the AI to generate vector representations of text.';
$string['selectcategory'] = '';
$string['selectcategory_help'] = '';

$string['aiproviderfeatures'] = '';
$string['aiproviderfeatures_desc'] = 'This plugin needs the following AI features';

Expand Down
3 changes: 2 additions & 1 deletion lib/db/upgrade.php
Original file line number Diff line number Diff line change
Expand Up @@ -1167,9 +1167,10 @@ function xmldb_main_upgrade($oldversion) {
upgrade_main_savepoint(true, 2024041200.00);
}

$table = new xmldb_table('aiprovider');
if (!$dbman->table_exists($table)) {
// Define table aiprovider to be created.
$table = new xmldb_table('aiprovider');


// Adding fields to table aiprovider.
$table->add_field('id', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, XMLDB_SEQUENCE, null);
Expand Down
14 changes: 10 additions & 4 deletions search/engine/solrrag/classes/engine.php
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
namespace search_solrrag;

use core_ai\api;
use core_ai\logger;
use search_solrrag\document;
use search_solrrag\schema;
//require_once($CFG->dirroot . "/search/engine/solrrag/lib.php");
Expand All @@ -20,7 +21,7 @@ class engine extends \search_solr\engine {
*/
protected ?AIClient $aiclient = null;
protected ?AIProvider $aiprovider = null;

protected ?logger $logger = null;
public function __construct(bool $alternateconfiguration = false)
{
parent::__construct($alternateconfiguration);
Expand Down Expand Up @@ -345,7 +346,9 @@ public function execute_query($filters, $accessinfo, $limit = 0) {
// We may get accessinfo, but we actually should determine our own ones to apply too
// But we can't access the "manager" class' get_areas_user_accesses function, and
// that's already been called based on the configuration / data from the user
return $this->execute_similarity_query($filters, $accessinfo, $limit);
$docs = $this->execute_similarity_query($filters, $accessinfo, $limit);
var_dump($docs);
return $docs;
} else {
// debugging("Running regular search", DEBUG_DEVELOPER);
// print_r($filters);
Expand Down Expand Up @@ -489,7 +492,10 @@ public function execute_similarity_query(\stdClass $filters, \stdClass $accessin
$requesturl = $this->get_connection_url('/select');
$requesturl->param('fl', 'id,areaid,score,content');
$requesturl->param('wt', 'xml');
$requesturl->param('fq', implode("&", $filterqueries));
foreach($filterqueries as $fq) {
$requesturl->param('fq', $fq);
}
// $requesturl->param('fq', implode("&", $filterqueries));

$params = [
"query" => $requestbody,
Expand All @@ -510,7 +516,7 @@ public function execute_similarity_query(\stdClass $filters, \stdClass $accessin
debugging($message, DEBUG_DEVELOPER);
} else if (isset($info['http_code']) && ($info['http_code'] !== 200)) {
// Unexpected HTTP response code.
$message = 'Error while indexing file with document id ' ;
$message = 'Error while querying for documents ' ;
// Try to get error message out of msg or title if it exists.
if (preg_match('|<str [^>]*name="msg"[^>]*>(.*?)</str>|i', $result, $matches)) {
$message .= ': ' . $matches[1];
Expand Down

0 comments on commit 32728fe

Please sign in to comment.